diff options
780 files changed, 16462 insertions, 7495 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index ed2ab49554..777eebd2c3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,31 +1,59 @@ version: '{branch}.{build}' skip_tags: true -image: Visual Studio 2017 +image: Visual Studio 2019 configuration: Release platform: x64 clone_depth: 5 environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true CLCACHE_SERVER: 1 - PACKAGES: berkeleydb boost-filesystem boost-signals2 boost-test libevent openssl rapidcheck zeromq double-conversion PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.0/Qt5.9.7_ssl_x64_static_vs2017.zip' - QT_DOWNLOAD_HASH: 'D4D35B8112302B67E5610A03421BB3E43FE13F14D9A5F637C22AE60DCEC0E0F5' - QT_LOCAL_PATH: 'C:\Qt5.9.7_ssl_x64_static_vs2017' + QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' + QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' + QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' + VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' + VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' cache: -- C:\tools\vcpkg\installed +- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt - C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in -- C:\Qt5.9.7_ssl_x64_static_vs2017 +- C:\Qt5.9.8_x64_static_vs2019 install: - cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq -- cmd: echo set(VCPKG_BUILD_TYPE release) >> C:\tools\vcpkg\triplets\%PLATFORM%-windows-static.cmake -- cmd: vcpkg remove --outdated --recurse -- cmd: vcpkg install --triplet %PLATFORM%-windows-static %PACKAGES% > NUL +# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: +# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file +# will cause the appveyor cache rules to invalidate the directory) +# 2. If the directory is missing: +# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, +# b. Install the missing packages. +- ps: | + $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt + Write-Host "vcpkg list: $env:PACKAGES" + if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) { + cd c:\tools\vcpkg + $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. + git pull origin master + git checkout $env:VCPKG_COMMIT_ID + .\bootstrap-vcpkg.bat + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" + .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null + cd "$env:APPVEYOR_BUILD_FOLDER" + } + else { + Write-Host "required vcpkg packages already installed." + } + c:\tools\vcpkg\vcpkg integrate install before_build: - ps: clcache -M 536870912 +# Powershell block below is to download and extract the Qt static libraries. The pseudo code is: +# 1. If the Qt destination directory exists assume it is correct and do nothing. To +# force a fresh install of the packages delete the job's appveyor cache. +# 2. Otherwise: +# a. Download the zip file with the prebuilt Qt static libraries. +# b. Check that the downloaded file matches the expected hash. +# c. Extract the zip file to the specific destination path expected by the msbuild projects. - ps: | if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) { Write-Host "Downloading Qt binaries."; @@ -44,17 +72,10 @@ before_build: Write-Host "Qt binaries already present."; } - cmd: python build_msvc\msvc-autogen.py -- ps: $files = (Get-ChildItem -Recurse | where {$_.extension -eq ".vcxproj"}).FullName -- ps: for (${i} = 0; ${i} -lt ${files}.length; ${i}++) { - ${content} = (Get-Content ${files}[${i}]); - ${content} = ${content}.Replace("</RuntimeLibrary>", "</RuntimeLibrary><DebugInformationFormat>None</DebugInformationFormat>"); - ${content} = ${content}.Replace("<WholeProgramOptimization>true", "<WholeProgramOptimization>false"); - Set-Content ${files}[${i}] ${content}; - } - ps: Start-Process clcache-server - ps: fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache) build_script: -- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:n /nologo +- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:q /nologo after_build: - ps: fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance) - ps: clcache -z @@ -64,7 +85,8 @@ test_script: - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - ps: python test\util\bitcoin-util-test.py - cmd: python test\util\rpcauth-test.py -- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast +# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. +- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation artifacts: #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index 9464ec1685..f4a3878ed8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,7 +16,7 @@ task: ccache_cache: folder: "/tmp/ccache_dir" install_script: - - pkg install -y autoconf automake boost-libs git gmake libevent libtool openssl pkgconf python3 ccache + - pkg install -y autoconf automake boost-libs git gmake libevent libtool pkgconf python3 ccache - ./contrib/install_db4.sh $(pwd) - ccache --max-size=${CCACHE_SIZE} configure_script: @@ -45,8 +45,6 @@ task: folder: "/tmp/ccache_dir" depends_built_cache: folder: "/tmp/cirrus-ci-build/depends/built" - depends_sdk_cache: - folder: "/tmp/cirrus-ci-build/depends/sdk-sources" install_script: - apt-get update - apt-get -y install git bash ccache diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md new file mode 100644 index 0000000000..c441f7a307 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -0,0 +1,21 @@ +--- +name: Good first issue +about: '(Regular devs only): Suggest a new good first issue' +title: '' +labels: good first issue +assignees: '' + +--- + + +#### Useful skills: + +<!-- (For example, “C++11 std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) --> + +#### Want to work on this issue? + +The purpose of the `good first issue` label is to highlight which issues are suitable for a new contributor without a deep understanding of the codebase. + +You do not need to request permission to start working on this. You are encouraged to comment on the issue if you are planning to work on it. This will help other contributors monitor which issues are actively being addressed and is also an effective way to request assistance if and when you need it. + +For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. diff --git a/.gitignore b/.gitignore index 809e851ff1..db493811c5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,9 @@ src/bitcoind src/bitcoin-cli src/bitcoin-tx src/bitcoin-wallet +src/test/fuzz +!src/test/fuzz/*.* src/test/test_bitcoin -src/test/test_bitcoin_fuzzy src/qt/test/test_bitcoin-qt # autoreconf @@ -28,6 +29,7 @@ build-aux/m4/ltversion.m4 build-aux/missing build-aux/compile build-aux/test-driver +config.cache config.log config.status configure diff --git a/.travis.yml b/.travis.yml index 3ddafda6d2..fbc81b2614 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ dist: xenial os: linux language: minimal +arch: amd64 cache: ccache: true directories: @@ -42,11 +43,11 @@ before_cache: stages: - lint - test - - extended-lint env: global: - CI_RETRY_EXE="travis_retry" - - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." + - CI_WAIT="while sleep 500; do echo .; done" + - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source ./ci/test/00_setup_env.sh - set -o errexit; source ./ci/test/03_before_install.sh @@ -80,67 +81,63 @@ jobs: script: - set -o errexit; source ./ci/lint/06_script.sh - - stage: extended-lint - name: 'extended lint [runtime >= 60 seconds]' - env: - cache: false - language: python - python: '3.5' - install: - - set -o errexit; source ./ci/extended_lint/04_install.sh - before_script: - - set -o errexit; source ./ci/lint/05_before_script.sh - script: - - set -o errexit; source ./ci/extended_lint/06_script.sh - - stage: test - name: 'ARM [GOAL: install] [unit tests, no functional tests]' + name: 'ARM [GOAL: install] [buster] [unit tests, functional tests]' + arch: arm64 env: >- FILE_ENV="./ci/test/00_setup_env_arm.sh" + QEMU_USER_CMD="" # Can run the tests natively without qemu - stage: test - name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' + name: 'S390x [GOAL: install] [buster] [unit tests, functional tests]' + arch: s390x env: >- - FILE_ENV="./ci/test/00_setup_env_win64.sh" + FILE_ENV="./ci/test/00_setup_env_s390x.sh" + QEMU_USER_CMD="" # Can run the tests natively without qemu - stage: test - name: '32-bit + dash [GOAL: install] [gui]' + name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' env: >- - FILE_ENV="./ci/test/00_setup_env_i686.sh" + FILE_ENV="./ci/test/00_setup_env_win64.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]' + name: '32-bit + dash [GOAL: install] [CentOS 7] [gui]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_qt5.sh" + FILE_ENV="./ci/test/00_setup_env_i686_centos.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]' + name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package and some depends packages] [unsigned char]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_trusty.sh" + FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_tsan.sh" + FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" TEST_RUNNER_EXTRA="--exclude feature_block" # Not enough memory on travis machines - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_asan.sh" + FILE_ENV="./ci/test/00_setup_env_native_asan.sh" + + - stage: test + name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, valgrind]' + env: >- + FILE_ENV="./ci/test/00_setup_env_native_valgrind.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_fuzz.sh" + FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_nowallet.sh" + FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" - stage: test - name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' + name: 'macOS 10.12 [GOAL: deploy] [no functional tests]' env: >- FILE_ENV="./ci/test/00_setup_env_mac.sh" @@ -148,8 +145,8 @@ jobs: name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]' os: osx # Use the most recent version: - # Xcode 11, macOS 10.14, JDK 12.0.1 + # Xcode 11.2.1, macOS 10.14, JDK 13.0.1, SDK 10.15 # https://docs.travis-ci.com/user/reference/osx/#macos-version - osx_image: xcode11 + osx_image: xcode11.2 env: >- FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecb5704a8f..33c797d799 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ release cycle, overall merging, moderation and appointment of maintainers. If you're looking for somewhere to start contributing, check out the [good first issue](https://github.com/bitcoin/bitcoin/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) -list. +list or participate in a weekly [Bitcoin Core PR Review Club](https://bitcoincore.reviews/) meeting. Communication Channels ---------------------- @@ -88,10 +88,10 @@ the pull request affects. Valid areas as: - `refactor` for structural changes that do not change behavior - `rpc`, `rest` or `zmq` for changes to the RPC, REST or ZMQ APIs - `script` for changes to the scripts and tools - - `test` for changes to the bitcoin unit tests or QA tests + - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code - - `build` for changes to the GNU Autotools, reproducible builds or CI code + - `build` for changes to the GNU Autotools or reproducible builds Examples: @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2019 The Bitcoin Core developers -Copyright (c) 2009-2019 Bitcoin Developers +Copyright (c) 2009-2020 The Bitcoin Core developers +Copyright (c) 2009-2020 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.am b/Makefile.am index deae2c1b57..22b83e80dd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -25,7 +25,7 @@ BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT) -BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT) +BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win64-setup$(EXEEXT) empty := space := $(empty) $(empty) @@ -99,7 +99,7 @@ $(OSX_APP)/Contents/PkgInfo: $(OSX_APP)/Contents/Resources/empty.lproj: $(MKDIR_P) $(@D) - @touch $@ + @touch $@ $(OSX_APP)/Contents/Info.plist: $(OSX_PLIST) $(MKDIR_P) $(@D) @@ -189,7 +189,16 @@ $(BITCOIN_WALLET_BIN): FORCE $(MAKE) -C src $(@F) if USE_LCOV -LCOV_FILTER_PATTERN=-p "/usr/include/" -p "/usr/lib/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1" +LCOV_FILTER_PATTERN = \ + -p "/usr/include/" \ + -p "/usr/lib/" \ + -p "/usr/lib64/" \ + -p "src/leveldb/" \ + -p "src/bench/" \ + -p "src/univalue" \ + -p "src/crypto/ctaes" \ + -p "src/secp256k1" \ + -p "depends" baseline.info: $(LCOV) -c -i -d $(abs_builddir)/src -o $@ @@ -321,4 +330,3 @@ clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff - @@ -12,7 +12,7 @@ with no central authority: managing transactions and issuing money are carried out collectively by the network. Bitcoin Core is the name of open source software which enables the use of this currency. -For more information, as well as an immediately useable, binary version of +For more information, as well as an immediately usable, binary version of the Bitcoin Core software, see https://bitcoincore.org/en/download/, or read the [original whitepaper](https://bitcoincore.org/bitcoin.pdf). diff --git a/autogen.sh b/autogen.sh index 3e922e7e64..de16260b56 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2013-2016 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index d579dc2ed5..83d054af5f 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,8 +116,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test "x$bitcoin_cv_static_qt" = xyes; then _BITCOIN_QT_FIND_STATIC_PLUGINS AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + if test "x$TARGET_OS" != xandroid; then + _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) + AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + fi if test "x$TARGET_OS" = xwindows; then _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows]) AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) @@ -128,6 +130,9 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AX_CHECK_LINK_FLAG([[-framework IOKit]],[QT_LIBS="$QT_LIBS -framework IOKit"],[AC_MSG_ERROR(could not iokit framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin)],[-lqcocoa]) AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) + elif test "x$TARGET_OS" = xandroid; then + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype -lQt5EglSupport $QT_LIBS" + AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS @@ -345,6 +350,9 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/accessible"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" fi + if test -d "$qt_plugin_path/platforms/android"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" + fi if test "x$use_pkgconfig" = xyes; then : dnl m4_ifdef([PKG_CHECK_MODULES],[ diff --git a/build_msvc/README.md b/build_msvc/README.md index 88b1f514bd..e5aaf57abf 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -12,7 +12,7 @@ Quick Start The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. ``` -vcpkg install --triplet x64-windows-static boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb rapidcheck double-conversion +vcpkg install --triplet x64-windows-static berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent[thread] rapidcheck zeromq double-conversion py -3 build_msvc\msvc-autogen.py msbuild /m build_msvc\bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` @@ -33,56 +33,53 @@ The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/d - Boost - DoubleConversion - libevent -- OpenSSL - Qt5 - RapidCheck - ZeroMQ Qt --------------------- -All the Bitcoin Core applications are configured to build with static linking. In order to build the Bitcoin Core Qt applications a static build of Qt is required. +In order to build the Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. -The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. OpenSSL must also be linked into the Qt binaries in order to provide full functionality of the Bitcoin Core Qt programs. An example of the configure command to build Qtv5.9.7 locally to link with Bitcoin Core is shown below (adjust paths accordingly), note it can be expected that the configure and subsequent build will fail numerous times until dependency issues are resolved. +Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. -```` -..\Qtv5.9.7_src\configure -developer-build -confirm-license -debug-and-release -opensource -platform win32-msvc -opengl desktop -no-shared -static -no-static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -ltcg -make libs -make tools -no-libjpeg -nomake examples -no-compile-examples -no-dbus -no-libudev -no-qml-debug -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -nomake tests -openssl-linked -IC:\Dev\github\vcpkg\installed\x64-windows-static\include -LC:\Dev\github\vcpkg\installed\x64-windows-static\lib OPENSSL_LIBS="-llibeay32 -lssleay32 -lgdi32 -luser32 -lwsock32 -ladvapi32" -prefix C:\Qt5.9.7_ssl_x64_static_vs2017 -```` +To determine which Qt prebuilt version to download open the `.appveyor.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths. -A prebuilt version for x64 and Visual C++ runtime v141 (Visual Studio 2017) can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware this download is NOT an officially sanctioned Bitcoin Core distribution and is provided for developer convenience. It should NOT be used for builds that will be used in a production environment or with real funds. - -To build Bitcoin Core without Qt unload or disable the bitcoin-qt, libbitcoin_qt and test_bitcoin-qt projects. +To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. Building --------------------- The instructions below use `vcpkg` to install the dependencies. -- Clone `vcpkg` from the [github repository](https://github.com/Microsoft/vcpkg) and install as per the instructions in the main README.md. -- Install the required packages (replace x64 with x86 as required): +- Install [`vcpkg`](https://github.com/Microsoft/vcpkg). +- Install the required packages (replace x64 with x86 as required). The list of required packages can be found in the `build_msvc\vcpkg-packages.txt` file. The PowerShell command below will work if run from the repository root directory and `vcpkg` is in the path. Alternatively the contents of the packages text file can be pasted in place of the `Get-Content` cmdlet. ``` - PS >.\vcpkg install --triplet x64-windows-static boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb rapidcheck double-conversion +PS >.\vcpkg install --triplet x64-windows-static $(Get-Content -Path build_msvc\vcpkg-packages.txt).split() ``` -- Use Python to generate *.vcxproj from Makefile +- Use Python to generate `*.vcxproj` from Makefile ``` - PS >py -3 msvc-autogen.py +PS >py -3 msvc-autogen.py ``` - An optional step is to adjust the settings in the build_msvc directory and the common.init.vcxproj file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. -- Build with Visual Studio 2017 or msbuild. +- To build from the command line with the Visual Studio 2017 toolchain use: ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build +msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build ``` -- Build with Visual Studio 2019 or msbuild. +- To build from the command line with the Visual Studio 2019 toolchain use: ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v142 /t:build +msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` +- Alternatively open the `build_msvc\bitcoin.sln` file in Visual Studio. + AppVeyor --------------------- The .appveyor.yml in the root directory is suitable to perform builds on [AppVeyor](https://www.appveyor.com/) Continuous Integration servers. The simplest way to perform an AppVeyor build is to fork Bitcoin Core and then configure a new AppVeyor Project pointing to the forked repository. diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in index 56401d7618..128c1bd8e7 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in @@ -42,6 +42,9 @@ <ProjectReference Include="..\libleveldb\libleveldb.vcxproj"> <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> </ProjectReference> + <ProjectReference Include="..\libtest_util\libtest_util.vcxproj"> + <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project> + </ProjectReference> </ItemGroup> <Target Name="RawBenchHeaderGen" BeforeTargets="PrepareForBuild"> <PropertyGroup> diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index fdeec55ee8..17cd31a52e 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -50,12 +50,13 @@ </ProjectReference> </ItemGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> <ClCompile> <AdditionalIncludeDirectories>$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <AdditionalDependencies>$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> @@ -63,12 +64,13 @@ </ResourceCompile> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> <ClCompile> <AdditionalIncludeDirectories>$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index d4b83b6529..5e9715451f 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -44,6 +44,10 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_qt", "libbitcoin EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-qt", "bitcoin-qt\bitcoin-qt.vcxproj", "{7E99172D-7FF2-4CB6-B736-AC9B76ED412A}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_util\libtest_util.vcxproj", "{868474FD-35F6-4400-8EED-30A33E7521D4}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -220,11 +224,29 @@ Global {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.Build.0 = Release|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.ActiveCfg = Release|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.Build.0 = Release|Win32 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.ActiveCfg = Debug|x64 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.Build.0 = Debug|x64 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.ActiveCfg = Debug|Win32 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.Build.0 = Debug|Win32 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.ActiveCfg = Release|x64 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.Build.0 = Release|x64 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.ActiveCfg = Release|Win32 + {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.Build.0 = Release|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.ActiveCfg = Debug|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.ActiveCfg = Debug|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.Build.0 = Debug|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.ActiveCfg = Release|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9} + SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9} + SolutionGuid = {8607C0F4-F33D-41B8-8D51-18E366A0F8DF} + SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70} EndGlobalSection EndGlobal diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index e892765fde..3178f2a3d8 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -1,3 +1,7 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_BITCOIN_CONFIG_H #define BITCOIN_BITCOIN_CONFIG_H @@ -49,9 +53,6 @@ /* define if the Boost::Filesystem library is available */ #define HAVE_BOOST_FILESYSTEM /**/ -/* define if the Boost::PROGRAM_OPTIONS library is available */ -#define HAVE_BOOST_PROGRAM_OPTIONS /**/ - /* define if the Boost::System library is available */ #define HAVE_BOOST_SYSTEM /**/ @@ -179,75 +180,6 @@ /* Define to 1 if you have the <inttypes.h> header file. */ #define HAVE_INTTYPES_H 1 -/* Define to 1 if you have the `advapi32' library (-ladvapi32). */ -#define HAVE_LIBADVAPI32 1 - -/* Define to 1 if you have the `comctl32' library (-lcomctl32). */ -#define HAVE_LIBCOMCTL32 1 - -/* Define to 1 if you have the `comdlg32' library (-lcomdlg32). */ -#define HAVE_LIBCOMDLG32 1 - -/* Define to 1 if you have the `crypt32' library (-lcrypt32). */ -#define HAVE_LIBCRYPT32 1 - -/* Define to 1 if you have the `gdi32' library (-lgdi32). */ -#define HAVE_LIBGDI32 1 - -/* Define to 1 if you have the `imm32' library (-limm32). */ -#define HAVE_LIBIMM32 1 - -/* Define to 1 if you have the `iphlpapi' library (-liphlpapi). */ -#define HAVE_LIBIPHLPAPI 1 - -/* Define to 1 if you have the `kernel32' library (-lkernel32). */ -#define HAVE_LIBKERNEL32 1 - -/* Define to 1 if you have the `mingwthrd' library (-lmingwthrd). */ -#define HAVE_LIBMINGWTHRD 1 - -/* Define to 1 if you have the `mswsock' library (-lmswsock). */ -#define HAVE_LIBMSWSOCK 1 - -/* Define to 1 if you have the `ole32' library (-lole32). */ -#define HAVE_LIBOLE32 1 - -/* Define to 1 if you have the `oleaut32' library (-loleaut32). */ -#define HAVE_LIBOLEAUT32 1 - -/* Define to 1 if you have the `rpcrt4' library (-lrpcrt4). */ -#define HAVE_LIBRPCRT4 1 - -/* Define to 1 if you have the `rt' library (-lrt). */ -/* #undef HAVE_LIBRT */ - -/* Define to 1 if you have the `shell32' library (-lshell32). */ -#define HAVE_LIBSHELL32 1 - -/* Define to 1 if you have the `shlwapi' library (-lshlwapi). */ -#define HAVE_LIBSHLWAPI 1 - -/* Define to 1 if you have the `ssp' library (-lssp). */ -#define HAVE_LIBSSP 1 - -/* Define to 1 if you have the `user32' library (-luser32). */ -#define HAVE_LIBUSER32 1 - -/* Define to 1 if you have the `uuid' library (-luuid). */ -#define HAVE_LIBUUID 1 - -/* Define to 1 if you have the `winmm' library (-lwinmm). */ -#define HAVE_LIBWINMM 1 - -/* Define to 1 if you have the `winspool' library (-lwinspool). */ -#define HAVE_LIBWINSPOOL 1 - -/* Define to 1 if you have the `ws2_32' library (-lws2_32). */ -#define HAVE_LIBWS2_32 1 - -/* Define to 1 if you have the `z ' library (-lz ). */ -#define HAVE_LIBZ_ 1 - /* Define this symbol if you have malloc_info */ /* #undef HAVE_MALLOC_INFO */ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 77f6a5c621..c09997d39d 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -6,6 +6,7 @@ <VCProjectVersion>16.0</VCProjectVersion> <VcpkgTriplet Condition="'$(Platform)'=='Win32'">x86-windows-static</VcpkgTriplet> <VcpkgTriplet Condition="'$(Platform)'=='x64'">x64-windows-static</VcpkgTriplet> + <UseNativeEnvironment>true</UseNativeEnvironment> </PropertyGroup> <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'=='' and !Exists('$(WindowsSdkDir)\DesignTime\CommonConfiguration\Neutral\Windows.props')"> @@ -39,7 +40,7 @@ <LinkIncremental>true</LinkIncremental> <WholeProgramOptimization>false</WholeProgramOptimization> <UseDebugLibraries>true</UseDebugLibraries> - <PlatformToolset>v141</PlatformToolset> + <PlatformToolset>v142</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> @@ -48,7 +49,7 @@ <LinkIncremental>false</LinkIncremental> <WholeProgramOptimization>true</WholeProgramOptimization> <UseDebugLibraries>false</UseDebugLibraries> - <PlatformToolset>v141</PlatformToolset> + <PlatformToolset>v142</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> @@ -107,7 +108,7 @@ <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions> - <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;</DisableSpecificWarnings> + <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> <PreprocessorDefinitions>ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> @@ -115,7 +116,7 @@ <Link> <SubSystem>Console</SubSystem> <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>crypt32.lib;Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> <Lib> <AdditionalOptions>/ignore:4221</AdditionalOptions> diff --git a/build_msvc/common.qt.init.vcxproj b/build_msvc/common.qt.init.vcxproj index e21288e26b..42150a2310 100644 --- a/build_msvc/common.qt.init.vcxproj +++ b/build_msvc/common.qt.init.vcxproj @@ -2,7 +2,7 @@ <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <PropertyGroup Label="QtGlobals"> - <QtBaseDir>C:\Qt5.9.7_ssl_x64_static_vs2017</QtBaseDir> + <QtBaseDir>C:\Qt5.9.8_x64_static_vs2019</QtBaseDir> <QtPluginsLibraryDir>$(QtBaseDir)\plugins</QtPluginsLibraryDir> <QtLibraryDir>$(QtBaseDir)\lib</QtLibraryDir> <QtIncludeDir>$(QtBaseDir)\include</QtIncludeDir> diff --git a/build_msvc/libtest_util/libtest_util.vcxproj.in b/build_msvc/libtest_util/libtest_util.vcxproj.in new file mode 100644 index 0000000000..b5e844010e --- /dev/null +++ b/build_msvc/libtest_util/libtest_util.vcxproj.in @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="..\common.init.vcxproj" /> + <PropertyGroup Label="Globals"> + <ProjectGuid>{868474FD-35F6-4400-8EED-30A33E7521D4}</ProjectGuid> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <ConfigurationType>StaticLibrary</ConfigurationType> + </PropertyGroup> + <ItemGroup> +@SOURCE_FILES@ + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <Import Project="..\common.vcxproj" /> +</Project> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index 5ddda3c03e..d99b17d381 100644 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2016-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import os import re @@ -18,6 +21,7 @@ libs = [ 'libbitcoin_wallet', 'libbitcoin_zmq', 'bench_bitcoin', + 'libtest_util', ] ignore_list = [ diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 8e54bc7653..2095c0c321 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -1,16 +1,14 @@ <?xml version="1.0" encoding="utf-8"?> -<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Import Project="..\common.init.vcxproj" /> <Import Project="..\common.qt.init.vcxproj" /> <PropertyGroup Label="Globals"> <ProjectGuid>{51201D5E-D939-4854-AE9D-008F03FF518E}</ProjectGuid> - </PropertyGroup> - <PropertyGroup Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> <ItemGroup> - <ClCompile Include="..\..\src\test\setup_common.cpp" /> + <ClCompile Include="..\..\src\test\util\setup_common.cpp" /> <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" /> <ClCompile Include="..\..\src\qt\test\apptests.cpp" /> <ClCompile Include="..\..\src\qt\test\compattests.cpp" /> @@ -66,29 +64,32 @@ </ProjectReference> </ItemGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> <ClCompile> <AdditionalIncludeDirectories>..\libbitcoin_qt\$(GeneratedFilesOutDir)\..\;$(QtIncludeDir)\QtTest;$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> - <AdditionalDependencies>$(QtReleaseLibaries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies>$(QtLibraryDir)\Qt5Test.lib;$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> <ClCompile> <AdditionalIncludeDirectories>..\libbitcoin_qt\$(GeneratedFilesOutDir)\..\;$(QtIncludeDir)\QtTest;$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> </ItemDefinitionGroup> - <ItemGroup> <MocTestFiles Include="..\..\src\qt\test\addressbooktests.h" /> <MocTestFiles Include="..\..\src\qt\test\apptests.h" /> <MocTestFiles Include="..\..\src\qt\test\compattests.h" /> - <MocTestFiles Include="..\..\src\qt\test\paymentservertests.h" /> <MocTestFiles Include="..\..\src\qt\test\rpcnestedtests.h" /> <MocTestFiles Include="..\..\src\qt\test\uritests.h" /> <MocTestFiles Include="..\..\src\qt\test\wallettests.h" /> @@ -106,8 +107,6 @@ <RemoveDir Directories="$(GeneratedFilesOutDir)\moc\*" /> <RemoveDir Directories="$(GeneratedFilesOutDir)\moc" /> </Target> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <PropertyGroup> <BuildDependsOn> moccode; @@ -120,4 +119,4 @@ $(CleanDependsOn); </CleanDependsOn> </PropertyGroup> - </Project> +</Project> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 703f616f8e..5c4b777d51 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -9,14 +9,13 @@ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> <ItemGroup> - <ClCompile Include="..\..\src\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\*_properties.cpp" /> + <ClCompile Include="..\..\src\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> - <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> - <ClCompile Include="..\..\src\test\lib\*.cpp" /> - <ClCompile Include="..\..\src\test\setup_common.cpp" /> <ClCompile Include="..\..\src\test\main.cpp" /> + <ClCompile Include="..\..\src\test\util\*.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> + <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> @@ -43,6 +42,9 @@ <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj"> <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project> </ProjectReference> + <ProjectReference Include="..\libtest_util\libtest_util.vcxproj"> + <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project> + </ProjectReference> <ProjectReference Include="..\libunivalue\libunivalue.vcxproj"> <Project>{5724ba7d-a09a-4ba8-800b-c4c1561b3d69}</Project> </ProjectReference> @@ -59,7 +61,9 @@ </PropertyGroup> <ItemGroup> <JsonTestFile Include="..\..\src\test\data\*.json" /> + <RawTestFile Include="..\..\src\test\data\*.raw" /> </ItemGroup> + <HeaderFromHexdump RawFilePath="%(RawTestFile.FullPath)" HeaderFilePath="%(RawTestFile.FullPath).h" SourceHeader="static unsigned const char %(RawTestFile.Filename)_raw[] = {" SourceFooter="};" /> <HeaderFromHexdump RawFilePath="%(JsonTestFile.FullPath)" HeaderFilePath="%(JsonTestFile.FullPath).h" SourceHeader="namespace json_tests{ static unsigned const char %(JsonTestFile.Filename)[] = {" SourceFooter="};}" /> </Target> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp index 0068f588cc..571d19957f 100644 --- a/build_msvc/testconsensus/testconsensus.cpp +++ b/build_msvc/testconsensus/testconsensus.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <iostream> // bitcoin includes. diff --git a/build_msvc/vcpkg-packages.txt b/build_msvc/vcpkg-packages.txt new file mode 100644 index 0000000000..27e2bbdcde --- /dev/null +++ b/build_msvc/vcpkg-packages.txt @@ -0,0 +1 @@ +berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent[thread] rapidcheck zeromq double-conversion
\ No newline at end of file diff --git a/ci/README.md b/ci/README.md index 16c481158f..880e49b459 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,11 +8,21 @@ and numbered according to which stage and lifecycle step it belongs to. ### Running a stage locally +Be aware that the tests will be built and run in-place, so please run at your own risk. +If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. + +The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. +While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, +such as the depends cache or ccache, are mounted as read-write into the docker container. While it should be fine to run +the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and +testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci +system in a virtual machine with a Linux operating system of your choice. + To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage requires `docker` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io ccache bash git +sudo apt install docker.io bash ``` To run the default test stage, @@ -26,6 +36,3 @@ To run the test stage with a specific configuration, ``` FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` - -Be aware that the tests will be build and run in-place, so please run at your own risk. -If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. diff --git a/ci/extended_lint/04_install.sh b/ci/extended_lint/04_install.sh deleted file mode 100755 index 123d874a84..0000000000 --- a/ci/extended_lint/04_install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -CPPCHECK_VERSION=1.86 -curl -s https://codeload.github.com/danmar/cppcheck/tar.gz/${CPPCHECK_VERSION} | tar -zxf - --directory /tmp/ -(cd /tmp/cppcheck-${CPPCHECK_VERSION}/ && make CFGDIR=/tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ > /dev/null) -export PATH="$PATH:/tmp/cppcheck-${CPPCHECK_VERSION}/" diff --git a/ci/extended_lint/06_script.sh b/ci/extended_lint/06_script.sh deleted file mode 100755 index e8228c9c4d..0000000000 --- a/ci/extended_lint/06_script.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -test/lint/extended-lint-all.sh diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 12c3bfce45..8b2d609504 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ export LC_ALL=C travis_retry pip3 install codespell==1.15.0 travis_retry pip3 install flake8==3.7.8 +travis_retry pip3 install yq SHELLCHECK_VERSION=v0.6.0 curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/lint/05_before_script.sh b/ci/lint/05_before_script.sh index 28bcbb47f7..2987812c8e 100755 --- a/ci/lint/05_before_script.sh +++ b/ci/lint/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index c7dea599dc..ae8122f9af 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/retry/README.md b/ci/retry/README.md index 983a498070..1b03c652db 100644 --- a/ci/retry/README.md +++ b/ci/retry/README.md @@ -32,8 +32,8 @@ Help: -v, --verbose Verbose output -t, --tries=# Set max retries: Default 10 -s, --sleep=secs Constant sleep amount (seconds) - -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 - -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -m, --min=secs Exponential Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponential Backoff: maximum sleep amount (seconds): Default 60 -f, --fail="script +cmds" Fail Script: run in case of final failure ### Examples diff --git a/ci/retry/retry b/ci/retry/retry index 0e5f6e9701..3c06519dbd 100755 --- a/ci/retry/retry +++ b/ci/retry/retry @@ -17,7 +17,7 @@ __log_out() { echo "$1" 1>&2 } -# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND +# Parameters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND retry() { local max_tries="$1"; shift @@ -83,8 +83,8 @@ Usage: $retry [options] -- execute command -v, --verbose Verbose output -t, --tries=# Set max retries: Default 10 -s, --sleep=secs Constant sleep amount (seconds) - -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 - -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -m, --min=secs Exponential Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponential Backoff: maximum sleep amount (seconds): Default 60 -f, --fail="script +cmds" Fail Script: run in case of final failure EOF } diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8bc50da2c3..a008d51523 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,6 +6,12 @@ export LC_ALL=C.UTF-8 +# The root dir. +# The ci system copies this folder. +# This is where the build is done (depends and dist). +BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_ROOT_DIR + echo "Setting specific values in env" if [ -n "${FILE_ENV}" ]; then set -o errexit; @@ -13,15 +19,16 @@ if [ -n "${FILE_ENV}" ]; then source "${FILE_ENV}" fi -BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) -export BASE_ROOT_DIR - echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py export MAKEJOBS=${MAKEJOBS:--j4} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) +# This folder only exists on the ci host. export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} -export HOST=${HOST:-x86_64-unknown-linux-gnu} +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} @@ -34,15 +41,18 @@ export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +# The cache dir. +# This folder exists on the ci host and ci guest. Changes are propagated back and forth. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} -# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo -export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} -# Folder where the build is done (bin and lib). Can not be changed. -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} +# The depends dir. +# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} +# Folder where the build is done (bin and lib). +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} export GOAL=${GOAL:-install} -export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} +export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH export CI_RETRY_EXE=${CI_RETRY_EXE:retry} diff --git a/ci/test/00_setup_env_amd64_asan.sh b/ci/test/00_setup_env_amd64_asan.sh deleted file mode 100644 index 46b870e145..0000000000 --- a/ci/test/00_setup_env_amd64_asan.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=x86_64-unknown-linux-gnu -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" -export NO_DEPENDS=1 -export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_amd64_trusty.sh b/ci/test/00_setup_env_amd64_trusty.sh deleted file mode 100644 index 51e98788c7..0000000000 --- a/ci/test/00_setup_env_amd64_trusty.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=x86_64-unknown-linux-gnu -export DOCKER_NAME_TAG=ubuntu:14.04 -export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev" -export NO_DEPENDS=1 -export RUN_FUNCTIONAL_TESTS=false -export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no" diff --git a/ci/test/00_setup_env_amd64_tsan.sh b/ci/test/00_setup_env_amd64_tsan.sh deleted file mode 100644 index 82ac988c41..0000000000 --- a/ci/test/00_setup_env_amd64_tsan.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=x86_64-unknown-linux-gnu -export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" -export NO_DEPENDS=1 -export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 9335f0b337..2a522f5a8f 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,11 +7,20 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -export QEMU_USER_CMD="qemu-arm -L /usr/arm-linux-gnueabihf/" -export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user" +# The host arch is unknown, so we run the tests through qemu. +# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. +if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi +export DPKG_ADD_ARCH="armhf" +export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +if [ -n "$QEMU_USER_CMD" ]; then + # Likely cross-compiling, so install the needed gcc and qemu-user + export PACKAGES="$PACKAGES qemu-user" +fi +# Use debian to avoid 404 apt errors when cross compiling +export DOCKER_NAME_TAG="debian:buster" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false +export RUN_FUNCTIONAL_TESTS=true export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default diff --git a/ci/test/00_setup_env_i686.sh b/ci/test/00_setup_env_i686.sh deleted file mode 100644 index 6df65dd4a0..0000000000 --- a/ci/test/00_setup_env_i686.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=i686-pc-linux-gnu -export PACKAGES="g++-multilib python3-zmq" -export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" -export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh new file mode 100644 index 0000000000..b875dceef0 --- /dev/null +++ b/ci/test/00_setup_env_i686_centos.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=i686-pc-linux-gnu +export DOCKER_NAME_TAG=centos:7 +export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python36-zmq which patch lbzip2 dash" +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" +export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index f384ba9263..af166b6ca7 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin14 +export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" export OSX_SDK=10.11 export RUN_UNIT_TESTS=false diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index aa68a5700e..0b437a723f 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin14 +export HOST=x86_64-apple-darwin16 export BREW_PACKAGES="automake berkeley-db4 libtool boost miniupnpc pkg-config qt qrencode python3 ccache zeromq" export PIP_PACKAGES="zmq" export RUN_CI_ON_HOST=true diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh new file mode 100644 index 0000000000..2ffd3c5107 --- /dev/null +++ b/ci/test/00_setup_env_native_asan.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +# Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running on ppc64le +export NO_DEPENDS=1 +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-8 CXX=clang++-8" diff --git a/ci/test/00_setup_env_amd64_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 7bdfc2c320..b0405bb762 100644 --- a/ci/test/00_setup_env_amd64_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,11 +6,11 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu -export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" +export PACKAGES="clang-8 llvm-8 python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang-8 CXX=clang++-8" +# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64 diff --git a/ci/test/00_setup_env_amd64_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index d5a2ba3111..53348559be 100644 --- a/ci/test/00_setup_env_amd64_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="python3-zmq" export DEP_OPTS="NO_WALLET=1" export GOAL="install" diff --git a/ci/test/00_setup_env_amd64_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 55820ea835..622ec3cfe1 100644 --- a/ci/test/00_setup_env_amd64_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh new file mode 100644 index 0000000000..e9b7a7bba1 --- /dev/null +++ b/ci/test/00_setup_env_native_tsan.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export DOCKER_NAME_TAG=ubuntu:16.04 +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export NO_DEPENDS=1 +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" + +# xenial comes with old clang versions that can not parse the sanitizer suppressions files +# Remove unparseable lines as a hacky workaround +sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh new file mode 100644 index 0000000000..906ffd7d79 --- /dev/null +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" +export USE_VALGRIND=1 +export NO_DEPENDS=1 +export TEST_RUNNER_EXTRA="p2p_segwit.py" # Only run one test for now. TODO enable all and bump timeouts +export RUN_FUNCTIONAL_TESTS=true +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh new file mode 100644 index 0000000000..6452feb5f2 --- /dev/null +++ b/ci/test/00_setup_env_s390x.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=s390x-linux-gnu +# The host arch is unknown, so we run the tests through qemu. +# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. +if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi +export PACKAGES="python3-zmq" +if [ -n "$QEMU_USER_CMD" ]; then + # Likely cross-compiling, so install the needed gcc and qemu-user + export DPKG_ADD_ARCH="s390x" + export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x" +fi +# Use debian to avoid 404 apt errors +export DOCKER_NAME_TAG="debian:buster" +export RUN_UNIT_TESTS=true +export RUN_FUNCTIONAL_TESTS=true +export GOAL="install" +export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb" diff --git a/ci/test/03_before_install.sh b/ci/test/03_before_install.sh index 5086114ba1..e939b9eeeb 100755 --- a/ci/test/03_before_install.sh +++ b/ci/test/03_before_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index d0831a4c13..4d5859e4d3 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,17 +1,25 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +if [[ $DOCKER_NAME_TAG == centos* ]]; then + export LC_ALL=en_US.utf8 +fi +if [[ $QEMU_USER_CMD == qemu-s390* ]]; then + export LC_ALL=C +fi + if [ "$TRAVIS_OS_NAME" == "osx" ]; then set +o errexit pushd /usr/local/Homebrew || exit 1 git reset --hard origin/master popd || exit 1 set -o errexit + ${CI_RETRY_EXE} brew unlink python@2 ${CI_RETRY_EXE} brew update # brew upgrade returns an error if any of the packages is already up to date # Failure is safe to ignore, unless we really need an update. @@ -25,61 +33,81 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then done export PATH="/usr/local/opt/ccache/libexec:$PATH" - OPENSSL_PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig" - export PKG_CONFIG_PATH=$OPENSSL_PKG_CONFIG_PATH:$PKG_CONFIG_PATH ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES fi mkdir -p "${BASE_SCRATCH_DIR}" -ccache echo "Creating ccache dir if it didn't already exist" +mkdir -p "${CCACHE_DIR}" -if [ ! -d ${DIR_QA_ASSETS} ]; then - git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} -fi -export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - -mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" -export ASAN_OPTIONS="detect_stack_use_after_return=1" -export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" -env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env +export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1" +export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" +env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) DOCKER_ADMIN="--cap-add SYS_PTRACE" fi +export P_CI_DIR="$PWD" + if [ -z "$RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ + --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ + --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ + -w $BASE_ROOT_DIR \ + --env-file /tmp/env \ + $DOCKER_NAME_TAG) DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" + docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } else echo "Running on host system without docker wrapper" DOCKER_EXEC () { - bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" + bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } fi +if [ -n "$DPKG_ADD_ARCH" ]; then + DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" +fi + +if [[ $DOCKER_NAME_TAG == centos* ]]; then + ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release + ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES +elif [ "$TRAVIS_OS_NAME" != "osx" ]; then + ${CI_RETRY_EXE} DOCKER_EXEC apt-get update + ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES +fi + if [ "$TRAVIS_OS_NAME" == "osx" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + DOCKER_EXEC echo "Free disk space:" + DOCKER_EXEC df -h +fi + +if [ ! -d ${DIR_QA_ASSETS} ]; then + DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ +DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [ "$TRAVIS_OS_NAME" != "osx" ]; then - ${CI_RETRY_EXE} DOCKER_EXEC apt-get update - ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES +if [ -z "$RUN_CI_ON_HOST" ]; then + echo "Create $BASE_ROOT_DIR" + DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR fi if [ "$USE_BUSY_BOX" = "true" ]; then diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index a0488f0807..d8aa5c87a2 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,17 +13,25 @@ else DOCKER_EXEC echo \> \$HOME/.bitcoin fi -mkdir -p depends/SDKs depends/sdk-sources +DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources -if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$OSX_SDK" ] && [ ! -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi -if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then - tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +if [ -n "$OSX_SDK" ] && [ -f ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + DOCKER_EXEC tar -C ${DEPENDS_DIR}/SDKs -xf ${DEPENDS_DIR}/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) fi if [ -z "$NO_DEPENDS" ]; then - DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + if [[ $DOCKER_NAME_TAG == centos* ]]; then + # CentOS has problems building the depends if the config shell is not explicitely set + # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to + # an error as the first command is executed) + SHELL_OPTS="CONFIG_SHELL=/bin/bash" + else + SHELL_OPTS="CONFIG_SHELL=" + fi + DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index c4dc22bdd8..b6043cd61e 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC ccache --max-size=$CCACHE_SIZE fi @@ -19,36 +19,28 @@ else fi END_FOLD -mkdir -p build - -# Temporarily disable errexit, because Travis macOS fails without error message -set +o errexit -cd build || (echo "could not enter build directory"; exit 1) -set -o errexit +DOCKER_EXEC mkdir -p build +export P_CI_DIR="$P_CI_DIR/build" BEGIN_FOLD configure -DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD BEGIN_FOLD distdir +# Create folder on host and docker, so that `cd` works +mkdir -p "bitcoin-$HOST" DOCKER_EXEC make distdir VERSION=$HOST END_FOLD -set +o errexit -cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1) -set -o errexit +export P_CI_DIR="$P_CI_DIR/bitcoin-$HOST" BEGIN_FOLD configure -DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) END_FOLD set -o errtrace -trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD - -set +o errexit -cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) -set -o errexit diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 048fe06c84..537493a710 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,39 +1,30 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -# Temporarily disable errexit, because Travis macOS fails without error message -set +o errexit -cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST"; exit 1) -set -o errexit - if [ -n "$QEMU_USER_CMD" ]; then BEGIN_FOLD wrap-qemu - echo "Prepare to run functional tests for HOST=$HOST" # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 - for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do - echo "Wrap $b ..." - DOCKER_EXEC mv "$b" "${b}_orig" - DOCKER_EXEC echo "\#\!/usr/bin/env bash" \> "$b" - DOCKER_EXEC echo "$QEMU_USER_CMD \\\"${b}_orig\\\" \\\"\\\$@\\\"" \>\> "$b" - DOCKER_EXEC chmod +x "$b" - done - done + DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" + END_FOLD +fi + +if [ -n "$USE_VALGRIND" ]; then + BEGIN_FOLD wrap-valgrind + DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" END_FOLD fi if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - bash -c "while sleep 500; do echo .; done" & # Print dots in case the unit tests take a long time to run - DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 + bash -c "${CI_WAIT}" & # Print dots in case the unit tests take a long time to run + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi @@ -48,7 +39,3 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN} END_FOLD fi - -set +o errexit -cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) -set -o errexit diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh new file mode 100755 index 0000000000..f1d3088748 --- /dev/null +++ b/ci/test/wrap-qemu.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do + # shellcheck disable=SC2044 + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + echo "Wrap $b ..." + mv "$b" "${b}_orig" + echo '#!/usr/bin/env bash' > "$b" + echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b" + chmod +x "$b" + done +done diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh new file mode 100755 index 0000000000..6b3e6eb7e7 --- /dev/null +++ b/ci/test/wrap-valgrind.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for b_name in "${BASE_OUTDIR}/bin"/*; do + # shellcheck disable=SC2044 + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + echo "Wrap $b ..." + mv "$b" "${b}_orig" + echo '#!/usr/bin/env bash' > "$b" + echo "valgrind --gen-suppressions=all --quiet --error-exitcode=1 --suppressions=${BASE_ROOT_DIR}/contrib/valgrind.supp \"${b}_orig\" \"\$@\"" >> "$b" + chmod +x "$b" + done +done diff --git a/configure.ac b/configure.ac index 9f2942dc9f..18f3104acb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,12 +1,11 @@ -dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) -AC_PREREQ([2.60]) +AC_PREREQ([2.69]) define(_CLIENT_VERSION_MAJOR, 0) define(_CLIENT_VERSION_MINOR, 19) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2019) +define(_COPYRIGHT_YEAR, 2020) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_REVISION, m4_if(_CLIENT_VERSION_BUILD, [0], [], _CLIENT_VERSION_BUILD))m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) @@ -243,27 +242,27 @@ AC_ARG_ENABLE(man, enable_man=yes) AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) -# Enable debug +dnl Enable debug AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], [use compiler flags and macros suited for debugging (default is no)])], [enable_debug=$enableval], [enable_debug=no]) -# Enable different -fsanitize options +dnl Enable different -fsanitize options AC_ARG_WITH([sanitizers], [AS_HELP_STRING([--with-sanitizers], [comma separated list of extra sanitizers to build with (default is none enabled)])], [use_sanitizers=$withval]) -# Enable gprof profiling +dnl Enable gprof profiling AC_ARG_ENABLE([gprof], [AS_HELP_STRING([--enable-gprof], [use gprof profiling compiler flags (default is no)])], [enable_gprof=$enableval], [enable_gprof=no]) -# Turn warnings into errors +dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], [Treat certain compiler warnings as errors (default is no)])], @@ -274,15 +273,15 @@ AC_LANG_PUSH([C++]) AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) if test "x$enable_debug" = xyes; then - # Clear default -g -O2 flags + dnl Clear default -g -O2 flags if test "x$CXXFLAGS_overridden" = xno; then CXXFLAGS="" fi - # Disable all optimizations + dnl Disable all optimizations AX_CHECK_COMPILE_FLAG([-O0], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]]) - # Prefer -g3, fall back to -g if that is unavailable. + dnl Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( [-g3], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"]], @@ -295,19 +294,19 @@ if test "x$enable_debug" = xyes; then fi if test x$use_sanitizers != x; then - # First check if the compiler accepts flags. If an incompatible pair like - # -fsanitize=address,thread is used here, this check will fail. This will also - # fail if a bad argument is passed, e.g. -fsanitize=undfeined + dnl First check if the compiler accepts flags. If an incompatible pair like + dnl -fsanitize=address,thread is used here, this check will fail. This will also + dnl fail if a bad argument is passed, e.g. -fsanitize=undfeined AX_CHECK_COMPILE_FLAG( [[-fsanitize=$use_sanitizers]], [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]], [AC_MSG_ERROR([compiler did not accept requested flags])]) - # Some compilers (e.g. GCC) require additional libraries like libasan, - # libtsan, libubsan, etc. Make sure linking still works with the sanitize - # flag. This is a separate check so we can give a better error message when - # the sanitize flags are supported by the compiler but the actual sanitizer - # libs are missing. + dnl Some compilers (e.g. GCC) require additional libraries like libasan, + dnl libtsan, libubsan, etc. Make sure linking still works with the sanitize + dnl flag. This is a separate check so we can give a better error message when + dnl the sanitize flags are supported by the compiler but the actual sanitizer + dnl libs are missing. AX_CHECK_LINK_FLAG( [[-fsanitize=$use_sanitizers]], [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]], @@ -329,6 +328,8 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -341,10 +342,12 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) - ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all - ## unknown options if any other warning is produced. Test the -Wfoo case, and - ## set the -Wno-foo case if it works. + dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all + dnl unknown options if any other warning is produced. Test the -Wfoo case, and + dnl set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) @@ -359,9 +362,9 @@ enable_shani=no if test "x$use_asm" = "xyes"; then -# Check for optional instruction set support. Enabling these does _not_ imply that all code will -# be compiled with them, rather that specific objects/libs may use them after checking for runtime -# compatibility. +dnl Check for optional instruction set support. Enabling these does _not_ imply that all code will +dnl be compiled with them, rather that specific objects/libs may use them after checking for runtime +dnl compatibility. AX_CHECK_COMPILE_FLAG([-msse4.2],[[SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]]) @@ -480,32 +483,27 @@ use_pkgconfig=yes case $host in *mingw*) - #pkgconfig does more harm than good with MinGW + dnl pkgconfig does more harm than good with MinGW use_pkgconfig=no TARGET_OS=windows - AC_CHECK_LIB([mingwthrd], [main],, AC_MSG_ERROR(libmingwthrd missing)) - AC_CHECK_LIB([kernel32], [main],, AC_MSG_ERROR(libkernel32 missing)) - AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) - AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) - AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) - AC_CHECK_LIB([winspool], [main],, AC_MSG_ERROR(libwinspool missing)) - AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) - AC_CHECK_LIB([shell32], [main],, AC_MSG_ERROR(libshell32 missing)) - AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) - AC_CHECK_LIB([ole32], [main],, AC_MSG_ERROR(libole32 missing)) - AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) - AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) - AC_CHECK_LIB([rpcrt4], [main],, AC_MSG_ERROR(librpcrt4 missing)) - AC_CHECK_LIB([advapi32], [main],, AC_MSG_ERROR(libadvapi32 missing)) - AC_CHECK_LIB([ws2_32], [main],, AC_MSG_ERROR(libws2_32 missing)) - AC_CHECK_LIB([mswsock], [main],, AC_MSG_ERROR(libmswsock missing)) - AC_CHECK_LIB([shlwapi], [main],, AC_MSG_ERROR(libshlwapi missing)) - AC_CHECK_LIB([iphlpapi], [main],, AC_MSG_ERROR(libiphlpapi missing)) - AC_CHECK_LIB([crypt32], [main],, AC_MSG_ERROR(libcrypt32 missing)) - - # -static is interpreted by libtool, where it has a different meaning. - # In libtool-speak, it's -all-static. + AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) + AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) + AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) + AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) + AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) + AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW],, AC_MSG_ERROR(libshell32 missing)) + AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) + AC_CHECK_LIB([ole32], [CoCreateInstance],, AC_MSG_ERROR(libole32 missing)) + AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) + AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) + AC_CHECK_LIB([advapi32], [CryptAcquireContextW],, AC_MSG_ERROR(libadvapi32 missing)) + AC_CHECK_LIB([ws2_32], [WSAStartup],, AC_MSG_ERROR(libws2_32 missing)) + AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW],, AC_MSG_ERROR(libshlwapi missing)) + AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses],, AC_MSG_ERROR(libiphlpapi missing)) + + dnl -static is interpreted by libtool, where it has a different meaning. + dnl In libtool-speak, it's -all-static. AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) AC_PATH_PROG([MAKENSIS], [makensis], none) @@ -523,12 +521,6 @@ case $host in if test "x$CXXFLAGS_overridden" = "xno"; then CXXFLAGS="$CXXFLAGS -w" fi - case $host in - i?86-*) WINDOWS_BITS=32 ;; - x86_64-*) WINDOWS_BITS=64 ;; - *) AC_MSG_ERROR("Could not determine win32/win64 for installer") ;; - esac - AC_SUBST(WINDOWS_BITS) dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -553,13 +545,8 @@ case $host in dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - openssl_prefix=`$BREW --prefix openssl 2>/dev/null` - bdb_prefix=`$BREW --prefix berkeley-db4 2>/dev/null` - qt5_prefix=`$BREW --prefix qt5 2>/dev/null` - if test x$openssl_prefix != x; then - PKG_CONFIG_PATH="$openssl_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" - export PKG_CONFIG_PATH - fi + bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) + qt5_prefix=$($BREW --prefix qt5 2>/dev/null) if test x$bdb_prefix != x; then CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" LIBS="$LIBS -L$bdb_prefix/lib" @@ -597,6 +584,7 @@ case $host in ;; *android*) dnl make sure android stays above linux for hosts like *linux-android* + TARGET_OS=android LEVELDB_TARGET_FLAGS="-DOS_ANDROID" ;; *linux*) @@ -675,11 +663,11 @@ AC_C_BIGENDIAN dnl Check for pthread compile/link requirements AX_PTHREAD -# The following macro will add the necessary defines to bitcoin-config.h, but -# they also need to be passed down to any subprojects. Pull the results out of -# the cache and add them to CPPFLAGS. +dnl The following macro will add the necessary defines to bitcoin-config.h, but +dnl they also need to be passed down to any subprojects. Pull the results out of +dnl the cache and add them to CPPFLAGS. AC_SYS_LARGEFILE -# detect POSIX or GNU variant of strerror_r +dnl detect POSIX or GNU variant of strerror_r AC_FUNC_STRERROR_R if test x$ac_cv_sys_file_offset_bits != x && @@ -702,12 +690,8 @@ AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then - #glibc absorbed clock_gettime in 2.17. librt (its previous location) is safe to link - #in anyway for back-compat. - AC_CHECK_LIB([rt],[clock_gettime],, AC_MSG_ERROR(librt missing)) - - #__fdelt_chk's params and return type have changed from long unsigned int to long int. - # See which one is present here. + dnl __fdelt_chk's params and return type have changed from long unsigned int to long int. + dnl See which one is present here. AC_MSG_CHECKING(__fdelt_chk type) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#ifdef _FORTIFY_SOURCE #undef _FORTIFY_SOURCE @@ -743,22 +727,22 @@ if test "x$enable_gprof" = xyes; then fi if test x$TARGET_OS != xwindows; then - # All windows code is PIC, forcing it on just adds useless compile warnings + dnl All windows code is PIC, forcing it on just adds useless compile warnings AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"]) fi -# All versions of gcc that we commonly use for building are subject to bug -# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set -# -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) +dnl All versions of gcc that we commonly use for building are subject to bug +dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set +dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) AX_CHECK_COMPILE_FLAG([-fstack-reuse=none],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) if test x$use_hardening != xno; then use_hardening=yes AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) - # When enable_debug is yes, all optimizations are disabled. - # However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. - # Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. + dnl When enable_debug is yes, all optimizations are disabled. + dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. + dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. if test x$enable_debug != xyes; then AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ @@ -785,12 +769,14 @@ fi dnl this flag screws up non-darwin gcc even when the check fails. special-case it. if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) + AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"]) + AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) -# FD_ZERO may be dependent on a declaration of memcpy, e.g. in SmartOS -# check that it fails to build without memcpy, then that it builds with +dnl FD_ZERO may be dependent on a declaration of memcpy, e.g. in SmartOS +dnl check that it fails to build without memcpy, then that it builds with AC_MSG_CHECKING(FD_ZERO memcpy dependence) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <cstddef> @@ -828,7 +814,7 @@ AC_CHECK_DECLS([getifaddrs, freeifaddrs],,, ) AC_CHECK_DECLS([strnlen]) -# Check for daemon(3), unrelated to --with-daemon (although used by it) +dnl Check for daemon(3), unrelated to --with-daemon (although used by it) AC_CHECK_DECLS([daemon]) AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, @@ -894,19 +880,19 @@ if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && te [ case $host in *mingw*) - # mingw32's implementation of thread_local has also been shown to behave - # erroneously under concurrent usage; see: - # https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 + dnl mingw32's implementation of thread_local has also been shown to behave + dnl erroneously under concurrent usage; see: + dnl https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 AC_MSG_RESULT(no) ;; *darwin*) - # TODO enable thread_local on later versions of Darwin where it is - # supported (per https://stackoverflow.com/a/29929949) + dnl TODO enable thread_local on later versions of Darwin where it is + dnl supported (per https://stackoverflow.com/a/29929949) AC_MSG_RESULT(no) ;; *freebsd*) - # FreeBSD's implementation of thread_local is also buggy (per - # https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) + dnl FreeBSD's implementation of thread_local is also buggy (per + dnl https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) AC_MSG_RESULT(no) ;; *) @@ -922,7 +908,7 @@ if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && te LDFLAGS="$TEMP_LDFLAGS" fi -# Check for different ways of gathering OS randomness +dnl Check for different ways of gathering OS randomness AC_MSG_CHECKING(for Linux getrandom syscall) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> #include <sys/syscall.h> @@ -947,6 +933,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> [ AC_MSG_RESULT(no)] ) +AC_MSG_CHECKING(for sysctl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> + #include <sys/sysctl.h>]], + [[ static const int name[2] = {CTL_KERN, KERN_VERSION}; + #ifdef __linux__ + #error "Don't use sysctl on Linux, it's deprecated even when it works" + #endif + sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #include <sys/sysctl.h>]], @@ -964,7 +962,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <stdint.h> [ AC_MSG_RESULT(no)] ) -# Check for reduced exports +dnl Check for reduced exports if test x$use_reduce_exports = xyes; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[RE_CXXFLAGS="-fvisibility=hidden"], [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) @@ -976,7 +974,7 @@ AC_LINK_IFELSE( [[ #include <cstdlib> ]], [[ int nErr = std::system(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_STD__SYSTEM, 1, Define to 1 if you have the `std::system' function.)], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_STD__SYSTEM, 1, Define to 1 if std::system is available.)], [ AC_MSG_RESULT(no) ] ) @@ -986,11 +984,10 @@ AC_LINK_IFELSE( [[ ]], [[ int nErr = ::_wsystem(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_WSYSTEM, 1, Define to 1 if you have the `::wsystem' function.)], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_WSYSTEM, 1, Define to 1 if ::wsystem is available.)], [ AC_MSG_RESULT(no) ] ) -# Define to 1 if std::system or ::wsystem (Windows) is available AC_DEFINE([HAVE_SYSTEM], [HAVE_STD__SYSTEM || HAVE_WSYSTEM], [std::system or ::wsystem]) LEVELDB_CPPFLAGS= @@ -1240,7 +1237,6 @@ if test x$use_pkgconfig = xyes; then m4_ifdef( [PKG_CHECK_MODULES], [ - PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi @@ -1263,8 +1259,6 @@ if test x$use_pkgconfig = xyes; then ] ) else - AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) - AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) @@ -1464,18 +1458,16 @@ if test x$bitcoin_enable_qt != xno; then AC_MSG_CHECKING([whether to build GUI with support for QR codes]) if test x$have_qrencode = xno; then if test x$use_qr = xyes; then - AC_MSG_ERROR("QR support requested but cannot be built. use --without-qrencode") + AC_MSG_ERROR([QR support requested but cannot be built. Use --without-qrencode]) fi - AC_MSG_RESULT(no) + use_qr=no else if test x$use_qr != xno; then - AC_MSG_RESULT(yes) AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) use_qr=yes - else - AC_MSG_RESULT(no) fi fi + AC_MSG_RESULT([$use_qr]) if test x$XGETTEXT = x; then AC_MSG_WARN("xgettext is required to update qt translations") @@ -1584,7 +1576,6 @@ AC_SUBST(TESTDEFS) AC_SUBST(LEVELDB_TARGET_FLAGS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) -AC_SUBST(CRYPTO_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) diff --git a/contrib/bitcoin-cli.bash-completion b/contrib/bitcoin-cli.bash-completion index f4cac84182..ddea58a05c 100644 --- a/contrib/bitcoin-cli.bash-completion +++ b/contrib/bitcoin-cli.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoin-cli(1) -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,13 +17,6 @@ _bitcoin_rpc() { $bitcoin_cli "${rpcargs[@]}" "$@" } -# Add wallet accounts to COMPREPLY -_bitcoin_accounts() { - local accounts - accounts=$(_bitcoin_rpc listaccounts | awk -F '"' '{ print $2 }') - COMPREPLY=( "${COMPREPLY[@]}" $( compgen -W "$accounts" -- "$cur" ) ) -} - _bitcoin_cli() { local cur prev words=() cword local bitcoin_cli @@ -60,10 +53,9 @@ _bitcoin_cli() { if ((cword > 3)); then case ${words[cword-3]} in addmultisigaddress) - _bitcoin_accounts return 0 ;; - getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaccount|listreceivedbyaddress|listsinceblock) + getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock) COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) return 0 ;; @@ -80,14 +72,10 @@ _bitcoin_cli() { COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) ) return 0 ;; - fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listaccounts|listreceivedbyaccount|listreceivedbyaddress|sendrawtransaction) + fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction) COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) return 0 ;; - move|setaccount) - _bitcoin_accounts - return 0 - ;; esac fi @@ -96,12 +84,11 @@ _bitcoin_cli() { _filedir return 0 ;; - getaddednodeinfo|getrawmempool|lockunspent|setgenerate) + getaddednodeinfo|getrawmempool|lockunspent) COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) return 0 ;; - getaccountaddress|getaddressesbyaccount|getbalance|getnewaddress|getreceivedbyaccount|listtransactions|move|sendfrom|sendmany) - _bitcoin_accounts + getbalance|getnewaddress|listtransactions|sendmany) return 0 ;; esac diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion index da869fa2c3..ec1d9512d4 100644 --- a/contrib/bitcoind.bash-completion +++ b/contrib/bitcoind.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoind(1) and bitcoin-qt(1) -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 0eccbacb96..581fe712e9 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com> Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2019, Bitcoin Core Developers +Copyright: 2009-2020, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 04fa02484f..515a0d8fc6 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -98,22 +98,26 @@ repository (requires pngcrush). security-check.py and test-security-check.py ============================================ -Perform basic ELF security checks on a series of executables. +Perform basic security checks on a series of executables. symbol-check.py =============== -A script to check that the (Linux) executables produced by gitian only contain -allowed gcc, glibc and libstdc++ version symbols. This makes sure they are -still compatible with the minimum supported Linux distribution versions. +A script to check that the executables produced by gitian only contain +certain symbols and are only linked against allowed libraries. + +For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. +This makes sure they are still compatible with the minimum supported distribution versions. + +For macOS we check that the executables are only linked against libraries we allow. Example usage after a gitian build: find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py -If only supported symbols are used the return value will be 0 and the output will be empty. +If no errors occur the return value will be 0 and the output will be empty. -If there are 'unsupported' symbols, the return value will be 1 a list like this will be printed: +If there are any errors the return value will be 1 and output like this will be printed: .../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14 .../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15 diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index 2e4657f1dd..6afa4351e7 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import sys import re diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 5307b04197..1b71245aab 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -19,6 +19,8 @@ EXCLUDE = [ 'src/qt/bitcoinstrings.cpp', 'src/chainparamsseeds.h', # other external copyrights: + 'src/reverse_iterator.h', + 'src/test/fuzz/FuzzedDataProvider.h', 'src/tinyformat.h', 'test/functional/test_framework/bignum.py', # python init: @@ -455,14 +457,14 @@ CPP_HEADER = ''' def get_cpp_header_lines_to_insert(start_year, end_year): return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) -PYTHON_HEADER = ''' +SCRIPT_HEADER = ''' # Copyright (c) %s The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -def get_python_header_lines_to_insert(start_year, end_year): - return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year)) +def get_script_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year)) ################################################################################ # query git for year of last change @@ -491,17 +493,18 @@ def file_has_hashbang(file_lines): return False return file_lines[0][:2] == '#!' -def insert_python_header(filename, file_lines, start_year, end_year): +def insert_script_header(filename, file_lines, start_year, end_year): if file_has_hashbang(file_lines): insert_idx = 1 else: insert_idx = 0 - header_lines = get_python_header_lines_to_insert(start_year, end_year) + header_lines = get_script_header_lines_to_insert(start_year, end_year) for line in header_lines: file_lines.insert(insert_idx, line) write_file_lines(filename, file_lines) def insert_cpp_header(filename, file_lines, start_year, end_year): + file_lines.insert(0, '\n') header_lines = get_cpp_header_lines_to_insert(start_year, end_year) for line in header_lines: file_lines.insert(0, line) @@ -513,8 +516,8 @@ def exec_insert_header(filename, style): sys.exit('*** %s already has a copyright by The Bitcoin Core developers' % (filename)) start_year, end_year = get_git_change_year_range(filename) - if style == 'python': - insert_python_header(filename, file_lines, start_year, end_year) + if style in ['python', 'shell']: + insert_script_header(filename, file_lines, start_year, end_year) else: insert_cpp_header(filename, file_lines, start_year, end_year) @@ -555,11 +558,13 @@ def insert_cmd(argv): if not os.path.isfile(filename): sys.exit("*** bad filename: %s" % filename) _, extension = os.path.splitext(filename) - if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: + if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']: sys.exit("*** cannot insert for file extension %s" % extension) if extension == '.py': style = 'python' + elif extension == '.sh': + style = 'shell' else: style = 'cpp' exec_insert_header(filename, style) diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index dbdb622877..aa65953d83 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2016-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 44b7f6c7cc..21d64e893d 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -Perform basic ELF security checks on a series of executables. +Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `readelf` (for ELF) and `objdump` (for PE). +Needs `readelf` (for ELF), `objdump` (for PE) and `otool` (for MACHO). ''' import subprocess import sys @@ -14,6 +14,7 @@ import os READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') +OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') NONFATAL = {} # checks which are non-fatal for now but only generate a warning def check_ELF_PIE(executable): @@ -162,6 +163,40 @@ def check_PE_NX(executable): (arch,bits) = get_PE_dll_characteristics(executable) return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT +def get_MACHO_executable_flags(executable): + p = subprocess.Popen([OTOOL_CMD, '-vh', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + + flags = [] + for line in stdout.splitlines(): + tokens = line.split() + # filter first two header lines + if 'magic' in tokens or 'Mach' in tokens: + continue + # filter ncmds and sizeofcmds values + flags += [t for t in tokens if not t.isdigit()] + return flags + +def check_MACHO_PIE(executable) -> bool: + ''' + Check for position independent executable (PIE), allowing for address space randomization. + ''' + flags = get_MACHO_executable_flags(executable) + if 'PIE' in flags: + return True + return False + +def check_MACHO_NOUNDEFS(executable) -> bool: + ''' + Check for no undefined references. + ''' + flags = get_MACHO_executable_flags(executable) + if 'NOUNDEFS' in flags: + return True + return False + CHECKS = { 'ELF': [ ('PIE', check_ELF_PIE), @@ -173,6 +208,10 @@ CHECKS = { ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_PE_NX) +], +'MACHO': [ + ('PIE', check_MACHO_PIE), + ('NOUNDEFS', check_MACHO_NOUNDEFS), ] } @@ -183,6 +222,8 @@ def identify_executable(executable): return 'PE' elif magic.startswith(b'\x7fELF'): return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' return None if __name__ == '__main__': diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index d8b684026c..f92d997621 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -4,8 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' A script to check that the (Linux) executables produced by gitian only contain -allowed gcc, glibc and libstdc++ version symbols. This makes sure they are -still compatible with the minimum supported Linux distribution versions. +allowed gcc and glibc version symbols. This makes sure they are still compatible +with the minimum supported Linux distribution versions. Example usage: @@ -15,31 +15,32 @@ import subprocess import re import sys import os +from typing import List, Optional, Tuple -# Debian 6.0.9 (Squeeze) has: +# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) -# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) +# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 10.04.4 (Lucid Lynx) has: +# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases # -# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) -# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) -# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) +# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial§ion=all) +# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial§ion=all) +# +# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General +# +# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) +# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) # # Taking the minimum of these as our target. # -# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: -# GCC 4.4.0: GCC_4.4.0 -# GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 -# (glibc) GLIBC_2_11 +# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: +# GCC 4.8.5: GCC_4.8.0 +# (glibc) GLIBC_2_17 # MAX_VERSIONS = { -'GCC': (4,4,0), -'CXXABI': (1,3,3), -'GLIBCXX': (3,4,13), -'GLIBC': (2,11), +'GCC': (4,8,0), +'GLIBC': (2,17), 'LIBATOMIC': (1,0) } # See here for a description of _IO_stdin_used: @@ -47,17 +48,19 @@ MAX_VERSIONS = { # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr' +'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', +'environ', '_environ', '__environ', } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') + # Allowed NEEDED libraries -ALLOWED_LIBRARIES = { +ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading -'libanl.so.1', # DNS resolve 'libm.so.6', # math library 'librt.so.1', # real-time (clock) 'libatomic.so.1', @@ -79,6 +82,25 @@ ARCH_MIN_GLIBC_VER = { 'AArch64':(2,17), 'RISC-V': (2,27) } + +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'libobjc.A.dylib', # Objective-C runtime library +} + class CPPFilt(object): ''' Demangle C++ symbol names. @@ -98,15 +120,15 @@ class CPPFilt(object): self.proc.stdout.close() self.proc.wait() -def read_symbols(executable, imports=True): +def read_symbols(executable, imports=True) -> List[Tuple[str, str, str]]: ''' - Parse an ELF executable and return a list of (symbol,version) tuples + Parse an ELF executable and return a list of (symbol,version, arch) tuples for dynamic, imported symbols. ''' p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: - raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) + raise IOError('Could not read symbols for {}: {}'.format(executable, stderr.strip())) syms = [] for line in stdout.splitlines(): line = line.split() @@ -121,7 +143,7 @@ def read_symbols(executable, imports=True): syms.append((sym, version, arch)) return syms -def check_version(max_versions, version, arch): +def check_version(max_versions, version, arch) -> bool: if '_' in version: (lib, _, ver) = version.rpartition('_') else: @@ -132,7 +154,7 @@ def check_version(max_versions, version, arch): return False return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] -def read_libraries(filename): +def elf_read_libraries(filename) -> List[str]: p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: @@ -148,26 +170,94 @@ def read_libraries(filename): raise ValueError('Unparseable (NEEDED) specification') return libraries -if __name__ == '__main__': +def check_imported_symbols(filename) -> bool: + cppfilt = CPPFilt() + ok = True + for sym, version, arch in read_symbols(filename, True): + if version and not check_version(MAX_VERSIONS, version, arch): + print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) + ok = False + return ok + +def check_exported_symbols(filename) -> bool: cppfilt = CPPFilt() + ok = True + for sym,version,arch in read_symbols(filename, False): + if arch == 'RISC-V' or sym in IGNORE_EXPORTS: + continue + print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + ok = False + return ok + +def check_ELF_libraries(filename) -> bool: + ok = True + for library_name in elf_read_libraries(filename): + if library_name not in ELF_ALLOWED_LIBRARIES: + print('{}: NEEDED library {} is not allowed'.format(filename, library_name)) + ok = False + return ok + +def macho_read_libraries(filename) -> List[str]: + p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.splitlines(): + tokens = line.split() + if len(tokens) == 1: # skip executable name + continue + libraries.append(tokens[0].split('/')[-1]) + return libraries + +def check_MACHO_libraries(filename) -> bool: + ok = True + for dylib in macho_read_libraries(filename): + if dylib not in MACHO_ALLOWED_LIBRARIES: + print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) + ok = False + return ok + +CHECKS = { +'ELF': [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries) +], +'MACHO': [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries) +] +} + +def identify_executable(executable) -> Optional[str]: + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' + return None + +if __name__ == '__main__': retval = 0 for filename in sys.argv[1:]: - # Check imported symbols - for sym,version,arch in read_symbols(filename, True): - if version and not check_version(MAX_VERSIONS, version, arch): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) - retval = 1 - # Check exported symbols - if arch != 'RISC-V': - for sym,version,arch in read_symbols(filename, False): - if sym in IGNORE_EXPORTS: - continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) - retval = 1 - # Check dependency libraries - for library_name in read_libraries(filename): - if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + try: + etype = identify_executable(filename) + if etype is None: + print('{}: unknown format'.format(filename)) retval = 1 + continue + failed = [] + for (name, func) in CHECKS[etype]: + if not func(filename): + failed.append(name) + if failed: + print('{}: failed {}'.format(filename, ' '.join(failed))) + retval = 1 + except IOError: + print('{}: cannot open'.format(filename)) + retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index bb864bfc0c..438d5f6bf0 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index 88ac850021..f5cd05a2c3 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -21,8 +21,8 @@ NON_DETERMINISTIC_TESTS=( "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() - "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 0000000000..dee25ff67b --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>' + echo + echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index df1db76e92..75034616f7 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2017-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index 570d4906cd..ade9e00d1b 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse import os @@ -51,10 +54,8 @@ def build(): os.chdir('gitian-builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) - subprocess.check_call(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) - subprocess.check_call(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) + subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) + subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) if args.linux: diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e0b9f74397..257dd8ba30 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -46,12 +46,12 @@ script: | FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS=-static-libstdc++ + HOST_LDFLAGS_BASE="-static-libstdc++" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -59,11 +59,12 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -77,7 +78,7 @@ script: | then echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} @@ -107,7 +108,7 @@ script: | rm -f ${WRAP_DIR}/${prog} cat << EOF > ${WRAP_DIR}/${prog} #!/usr/bin/env bash - REAL="`which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1`" + REAL="$(which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1)" for var in "\$@" do if [ "\$var" = "-m32" ]; then @@ -122,7 +123,7 @@ script: | done cd bitcoin - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" @@ -141,10 +142,11 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ make dist - SOURCEDIST=`echo bitcoin-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + SOURCEDIST=$(echo bitcoin-*.tar.gz) + DISTNAME=${SOURCEDIST/%.tar.gz} + # Correct tar file order mkdir -p temp pushd temp @@ -159,9 +161,16 @@ script: | # Extract the release tarball into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + if [ "${i}" = "riscv64-linux-gnu" ]; then + # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 + # TODO: remove this when no longer needed + HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" + else + HOST_LDFLAGS="${HOST_LDFLAGS_BASE}" + fi mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 4cfca403b1..a4f3219c22 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -17,14 +17,14 @@ script: | WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} - export PATH=`pwd`:$PATH + export PATH="$PWD":$PATH FAKETIME_PROGS="dmg genisoimage" # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index a563bef778..7c5abb9018 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -37,7 +37,7 @@ script: | set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin14" + HOSTS="x86_64-apple-darwin16" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" @@ -45,7 +45,7 @@ script: | export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -55,11 +55,12 @@ script: | export ZERO_AR_DATE=1 + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -71,7 +72,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} @@ -86,7 +87,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} cd bitcoin - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz @@ -104,10 +105,10 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ make dist - SOURCEDIST=`echo bitcoin-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + SOURCEDIST=$(echo bitcoin-*.tar.gz) + DISTNAME=${SOURCEDIST/%.tar.gz} # Correct tar file order mkdir -p temp @@ -125,7 +126,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST @@ -136,6 +137,8 @@ script: | CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} + make ${MAKEOPTS} -C src check-security + make ${MAKEOPTS} -C src check-symbols make install-strip DESTDIR=${INSTALLPATH} make osx_volname diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 656c6d9b7a..9d96465742 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -6,37 +6,36 @@ suites: architectures: - "amd64" packages: -# Once osslsigncode supports openssl 1.1, we can change this back to libssl-dev -- "libssl1.0-dev" +- "libssl-dev" - "autoconf" +- "libtool" +- "pkg-config" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" files: -- "osslsigncode-1.7.1.tar.gz" -- "osslsigncode-Backports-to-1.7.1.patch" +- "osslsigncode-2.0.tar.gz" - "bitcoin-win-unsigned.tar.gz" script: | set -e -o pipefail - BUILD_DIR=`pwd` + BUILD_DIR="$PWD" SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned - echo "f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 osslsigncode-1.7.1.tar.gz" | sha256sum -c - echo "a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 osslsigncode-Backports-to-1.7.1.patch" | sha256sum -c + echo "5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f osslsigncode-2.0.tar.gz" | sha256sum -c mkdir -p ${UNSIGNED_DIR} tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz - tar xf osslsigncode-1.7.1.tar.gz - cd osslsigncode-1.7.1 - patch -p1 < ${BUILD_DIR}/osslsigncode-Backports-to-1.7.1.patch + tar xf osslsigncode-2.0.tar.gz + cd osslsigncode-2.0 + ./autogen.sh ./configure --without-gsf --without-curl --disable-dependency-tracking make find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="`basename "${i}"`" - OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`" + INFILE="$(basename "${i}")" + OUTFILE="${INFILE/-unsigned}" ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index d600af84fb..de2e45190a 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -40,7 +40,7 @@ script: | export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 export TZ="UTC" - export BUILD_DIR=`pwd` + export BUILD_DIR="$PWD" mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} @@ -48,11 +48,12 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -64,7 +65,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} @@ -79,7 +80,7 @@ script: | for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} @@ -96,7 +97,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} cd bitcoin - BASEPREFIX=`pwd`/depends + BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" @@ -111,10 +112,11 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ + CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/ make dist - SOURCEDIST=`echo bitcoin-*.tar.gz` - DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` + SOURCEDIST=$(echo bitcoin-*.tar.gz) + DISTNAME=${SOURCEDIST/%.tar.gz} + # Correct tar file order mkdir -p temp pushd temp @@ -131,7 +133,7 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} - INSTALLPATH=`pwd`/installed/${DISTNAME} + INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST @@ -145,10 +147,7 @@ script: | make ${MAKEOPTS} -C src check-security make deploy make install DESTDIR=${INSTALLPATH} - ( - SETUP_EXE="$(basename "$(echo ./*-setup.exe)")" - cp -f "$SETUP_EXE" "${OUTDIR}/${SETUP_EXE/%-setup.exe/-setup-unsigned.exe}" - ) + cp -f --target-directory="${OUTDIR}" ./bitcoin-*-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 9222a40b17..ba3036a89f 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -27,6 +27,7 @@ D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy 37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgium) 133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille +A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 4dfa1729a5..46d755886c 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -62,15 +62,16 @@ Likewise, to perform a bootstrapped build (takes even longer): export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap --no-substitutes' ``` -### Using the right Guix +### Using a version of Guix with `guix time-machine` capabilities -Once Guix is installed, deploy our patched version into your current Guix -profile. The changes there are slowly being upstreamed. +> Note: This entire section can be skipped if you are already using a version of +> Guix that has [the `guix time-machine` command][guix/time-machine]. + +Once Guix is installed, if it doesn't have the `guix time-machine` command, pull +the latest `guix`. ```sh -guix pull --url=https://github.com/dongcarl/guix.git \ - --commit=82c77e52b8b46e0a3aad2cb12307c2e30547deec \ - --max-jobs=4 # change accordingly +guix pull --max-jobs=4 # change number of jobs accordingly ``` Make sure that you are using your current profile. (You are prompted to do this @@ -80,9 +81,6 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` -> Note: There is ongoing work to eliminate this entire section using Guix -> [inferiors][guix/inferiors] and [channels][guix/channels]. - ## Usage ### As a Development Environment @@ -224,6 +222,7 @@ repository and will likely put one up soon. [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html [guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html [guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html +[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 [fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index f8ba8c7ed2..5e0c681f29 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,6 +13,12 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +time-machine() { + guix time-machine --url=https://github.com/dongcarl/guix.git \ + --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + -- "$@" +} + # Deterministically build Bitcoin Core for HOSTs (overriable by environment) for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do @@ -22,18 +28,18 @@ for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64- # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm' # shellcheck disable=SC2086 - guix environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" done diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index ca11d7a0f0..23b656cad7 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -106,7 +106,7 @@ chain for " target " development.")) (base-libc glibc-2.27) (base-gcc (make-gcc-rpath-link (make-ssp-fixed-gcc gcc-9)))) - "Convienience wrapper around MAKE-CROSS-TOOLCHAIN with default values + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target base-gcc-for-libc diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 4b848dda99..e9130a21de 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. # Install libdb4.8 (Berkeley DB). diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 2315898bf1..5990b9307a 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -28,6 +28,11 @@ input=/home/example/.bitcoin/blocks #genesis=000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943 #input=/home/example/.bitcoin/testnet3/blocks +# regtest +#netmagic=fabfb5da +#genesis=0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206 +#input=/home/example/.bitcoin/regtest/blocks + # "output" option causes blockchain files to be written to the given location, # with "output_file" ignored. If not used, "output_file" is used instead. # output=/home/example/blockchain_directory diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 9ad7a03941..bcca3b7cea 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2018 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 02c96d2a75..fed6e665b8 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -2,7 +2,7 @@ # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # -# Copyright (c) 2013-2018 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index af2b11fa0d..5c5a85d3fe 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 938bcd1638..31a97f0a24 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index 4c175156f4..21243ada04 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index ccb0f4f895..8408545a21 100644 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index ec589d4c02..d516ca10c1 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2013-2018 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -19,13 +19,9 @@ MIN_BLOCKS = 337600 # These are hosts that have been observed to be behaving strangely (e.g. # aggressively connecting to every node). -SUSPICIOUS_HOSTS = { - "130.211.129.106", "178.63.107.226", - "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6", - "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211", - "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214", - "54.94.195.96", "54.94.200.247" -} +with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: + SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} + PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt new file mode 100644 index 0000000000..13385cc816 --- /dev/null +++ b/contrib/seeds/suspicious_hosts.txt @@ -0,0 +1,16 @@ +130.211.129.106 +148.251.238.178 +176.9.46.6 +178.63.107.226 +54.173.72.127 +54.174.10.182 +54.183.64.54 +54.194.231.211 +54.66.214.167 +54.66.220.137 +54.67.33.14 +54.77.251.214 +54.94.195.96 +54.94.200.247 +83.81.130.26 +88.198.17.7
\ No newline at end of file diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 0f6d993fd2..744b8ee70f 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -6,7 +6,14 @@ # Example use: # $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin # $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ -# --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +# --show-leak-kinds=all src/test/test_bitcoin +# +# To create suppressions for found issues, use the --gen-suppressions=all option: +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all --gen-suppressions=all --show-reachable=yes \ +# --error-limit=no src/test/test_bitcoin +# +# Note that suppressions may depend on OS and/or library versions. { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -23,8 +30,57 @@ Memcheck:Cond obj:*/libdb_cxx-*.so fun:__log_put +} +{ + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:__os_io +} +{ + Suppress libdb warning + Memcheck:Cond + fun:__log_putr.isra.1 +} +{ + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:__os_io + obj:*/libdb_cxx-*.so +} +{ + Suppress uninitialized bytes warning in compat code + Memcheck:Param + ioctl(TCSET{S,SW,SF}) + fun:tcsetattr +} +{ + Suppress libdb warning + Memcheck:Leak + fun:malloc + ... obj:*/libdb_cxx-*.so - fun:__log_put_record +} +{ + Suppress leaks on init + Memcheck:Leak + ... + fun:_Z11AppInitMainR11NodeContext +} +{ + Suppress leaks on shutdown + Memcheck:Leak + ... + fun:_Z8ShutdownR11NodeContext +} +{ + Ignore GUI warning + Memcheck:Leak + ... + obj:/usr/lib64/libgdk-3.so.0.2404.7 } { Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 @@ -41,3 +97,103 @@ ... fun:_ZN7leveldbL14InitDefaultEnvEv } +{ + Suppress leveldb leak + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldb6DBImpl14BackgroundCallEv +} +{ + Suppress leveldb leak + Memcheck:Leak + fun:_Znwm + ... + fun:GetCoin +} +{ + Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626) + Memcheck:Addr16 + fun:__wcsnlen_sse4_1 + fun:wcsnrtombs +} +{ + Suppress wcsnrtombs warning (remove after removing boost::fs) + Memcheck:Cond + ... + fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE + fun:unique_path +} +{ + Suppress boost warning + Memcheck:Leak + fun:_Znwm + ... + fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE + fun:_ZN5boost9unit_test9framework3runEmb + fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc + fun:main +} +{ + Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9) + Memcheck:Cond + fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE + ... + obj:*/libboost_filesystem.so.* +} +{ + Suppress boost::filesystem warning (could be related: https://stackoverflow.com/questions/9830182/function-boostfilesystemcomplete-being-reported-as-possible-memory-leak-by-v) + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_ +} +{ + Suppress boost still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_M_construct_aux<char*> + fun:_M_construct<char*> + fun:basic_string + fun:path +} +{ + Suppress LogInstance still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_Z11LogInstancev +} +{ + Suppress secp256k1_context_create still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:secp256k1_context_create +} +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} +{ + Suppress rest_blockhash_by_height Conditional jump or move depends on uninitialised value(s) + Memcheck:Cond + fun:_ZL24rest_blockhash_by_heightP11HTTPRequestRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE +} diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index 1215962a16..e95a57586f 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -40,7 +40,7 @@ Import trusted keys In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: ```sh -gpg --recv-keys $(<contrib/verify-commits/trusted-keys) +gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $(<contrib/verify-commits/trusted-keys) ``` Key expiry/revocation diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 288e4ccacb..db5bfce208 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index 4db4a90853..a26791f0d1 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 9ec8663fba..7e46c6fd47 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify commits against a trusted keys list.""" diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index bfe74aa4fa..4296998631 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index cc42422b23..31720e72e7 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/depends/.gitignore b/depends/.gitignore index 72734102c5..19c506ce54 100644 --- a/depends/.gitignore +++ b/depends/.gitignore @@ -8,5 +8,7 @@ i686* mips* arm* aarch64* +powerpc* riscv32* riscv64* +s390x* diff --git a/depends/Makefile b/depends/Makefile index 80df0e46f8..f81bc3001b 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -9,6 +9,7 @@ WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= +NO_QR ?= RAPIDCHECK ?= NO_WALLET ?= NO_ZMQ ?= @@ -58,6 +59,11 @@ full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) host_os:=$(findstring linux,$(full_host_os)) host_os+=$(findstring darwin,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) + +ifeq (android,$(findstring android,$(full_host_os))) +host_os:=android +endif + host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) @@ -95,7 +101,10 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) -qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) +qrencode_packages_$(NO_QR) = $(qrencode_packages) + +qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages) $(qrencode_packages_) + wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) @@ -149,6 +158,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ -e 's|@no_qt@|$(NO_QT)|' \ + -e 's|@no_qr@|$(NO_QR)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ @@ -181,7 +191,7 @@ $(host_prefix)/share/config.site: check-packages check-packages: check-sources clean-all: clean - @rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* riscv32* riscv64* + @rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x* clean: @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) diff --git a/depends/README.md b/depends/README.md index ca542be13f..93f619983f 100644 --- a/depends/README.md +++ b/depends/README.md @@ -12,26 +12,33 @@ For example: make HOST=x86_64-w64-mingw32 -j4 -**Bitcoin's configure script by default will ignore the depends output.** In +**Bitcoin Core's configure script by default will ignore the depends output.** In order for it to pick up libraries, tools, and settings from the depends build, you must point it at the appropriate `--prefix` directory generated by the build. In the above example, a prefix dir named x86_64-w64-mingw32 will be created. To use it for Bitcoin: - ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 + ./configure --prefix=$PWD/depends/x86_64-w64-mingw32 Common `host-platform-triplets` for cross compilation are: - `i686-pc-linux-gnu` for Linux 32 bit - `x86_64-pc-linux-gnu` for x86 Linux - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin14` for macOS +- `x86_64-apple-darwin16` for macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit +- `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian) +- `powerpc64le-linux-gnu` for Linux POWER 64-bit (little endian) - `riscv32-linux-gnu` for Linux RISC-V 32 bit - `riscv64-linux-gnu` for Linux RISC-V 64 bit +- `s390x-linux-gnu` for Linux S390X +- `armv7a-linux-android` for Android ARM 32 bit +- `aarch64-linux-android` for Android ARM 64 bit +- `i686-linux-android` for Android x86 32 bit +- `x86_64-linux-android` for Android x86 64 bit -No other options are needed, the paths are automatically configured. +The paths are automatically configured and no other options are needed unless targeting [Android](#Android). ### Install the required dependencies: Ubuntu & Debian @@ -57,6 +64,10 @@ For linux AARCH64 cross compilation: sudo apt-get install g++-aarch64-linux-gnu binutils-aarch64-linux-gnu +For linux POWER 64-bit cross compilation (there are no packages for 32-bit): + + sudo apt-get install g++-powerpc64-linux-gnu binutils-powerpc64-linux-gnu g++-powerpc64le-linux-gnu binutils-powerpc64le-linux-gnu + For linux RISC-V 64-bit cross compilation (there are no packages for 32-bit): sudo apt-get install g++-riscv64-linux-gnu binutils-riscv64-linux-gnu @@ -64,6 +75,10 @@ For linux RISC-V 64-bit cross compilation (there are no packages for 32-bit): RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_bitcoin` executable (see https://github.com/bitcoin/bitcoin/pull/13543), this is apparently fixed in gcc-8.1.0. +For linux S390X cross compilation: + + sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu + ### Dependency Options The following can be set when running make: make FOO=bar @@ -72,6 +87,7 @@ The following can be set when running make: make FOO=bar SDK_PATH: Path where sdk's can be found (used by macOS) FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up NO_QT: Don't download/build/cache qt and its dependencies + NO_QR: Don't download/build/cache packages needed for enabling qrencode NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq NO_WALLET: Don't download/build/cache libs needed to enable the wallet NO_UPNP: Don't download/build/cache packages needed for enabling upnp @@ -90,6 +106,19 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-win: run 'make download-win' to fetch all sources needed for win builds download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Android + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/depends/config.site.in b/depends/config.site.in index dee568bc25..c5731e5269 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -17,6 +17,10 @@ if test -z $with_qt_bindir && test -z "@no_qt@"; then with_qt_bindir=$depends_prefix/native/bin fi +if test -z $with_qrencode && test -n "@no_qr@"; then + with_qrencode=no +fi + if test -z $enable_wallet && test -n "@no_wallet@"; then enable_wallet=no fi diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk new file mode 100644 index 0000000000..969ec2a1cb --- /dev/null +++ b/depends/hosts/android.mk @@ -0,0 +1,11 @@ +ifeq ($(HOST),armv7a-linux-android) +android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar +android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ +android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib +else +android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar +android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ +android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib +endif diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index a1c943d60b..1f88c209cf 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=10.10 +OSX_MIN_VERSION=10.12 OSX_SDK_VERSION=10.11 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk LD64_VERSION=253.9 diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 766b8d224e..cd0e70fb1c 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -15,20 +15,25 @@ $(package)_config_opts_mingw32=binary-format=pe target-os=windows threadapi=win3 $(package)_config_opts_x86_64_mingw32=address-model=64 $(package)_config_opts_i686_mingw32=address-model=32 $(package)_config_opts_i686_linux=address-model=32 architecture=x86 +$(package)_config_opts_i686_android=address-model=32 +$(package)_config_opts_aarch64_android=address-model=64 +$(package)_config_opts_x86_64_android=address-model=64 +$(package)_config_opts_armv7a_android=address-model=32 $(package)_toolset_$(host_os)=gcc $(package)_archiver_$(host_os)=$($(package)_ar) $(package)_toolset_darwin=clang-darwin $(package)_config_libraries=chrono,filesystem,system,thread,test $(package)_cxxflags=-std=c++11 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC +$(package)_cxxflags_android=-fPIC endef define $(package)_preprocess_cmds - echo "using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$(boost_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam + echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam endef define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$(boost_config_libraries) + ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) endef define $(package)_build_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 300c806a8d..eb45e14f6f 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -1,8 +1,8 @@ package=libevent -$(package)_version=2.1.8-stable +$(package)_version=2.1.11-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz -$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d +$(package)_sha256_hash=229393ab2bf0dc94694f21836846b424f3532585bac3468738b7bf752c03901e define $(package)_preprocess_cmds ./autogen.sh @@ -13,6 +13,7 @@ define $(package)_set_vars $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic + $(package)_config_opts_android=--with-pic endef define $(package)_config_cmds diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk deleted file mode 100644 index 3498a3f6ee..0000000000 --- a/depends/packages/openssl.mk +++ /dev/null @@ -1,83 +0,0 @@ -package=openssl -$(package)_version=1.0.1k -$(package)_download_path=https://www.openssl.org/source -$(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=8f9faeaebad088e772f4ef5e38252d472be4d878c6b3a2718c10a4fcebe7a41c -$(package)_patches=0001-Add-OpenSSL-termios-fix-for-musl-libc.patch - -define $(package)_set_vars -$(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" -$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl -$(package)_config_opts+=no-camellia -$(package)_config_opts+=no-capieng -$(package)_config_opts+=no-cast -$(package)_config_opts+=no-comp -$(package)_config_opts+=no-dso -$(package)_config_opts+=no-dtls1 -$(package)_config_opts+=no-ec_nistp_64_gcc_128 -$(package)_config_opts+=no-gost -$(package)_config_opts+=no-gmp -$(package)_config_opts+=no-heartbeats -$(package)_config_opts+=no-idea -$(package)_config_opts+=no-jpake -$(package)_config_opts+=no-krb5 -$(package)_config_opts+=no-libunbound -$(package)_config_opts+=no-md2 -$(package)_config_opts+=no-mdc2 -$(package)_config_opts+=no-rc4 -$(package)_config_opts+=no-rc5 -$(package)_config_opts+=no-rdrand -$(package)_config_opts+=no-rfc3779 -$(package)_config_opts+=no-rsax -$(package)_config_opts+=no-sctp -$(package)_config_opts+=no-seed -$(package)_config_opts+=no-sha0 -$(package)_config_opts+=no-shared -$(package)_config_opts+=no-ssl-trace -$(package)_config_opts+=no-ssl2 -$(package)_config_opts+=no-ssl3 -$(package)_config_opts+=no-static_engine -$(package)_config_opts+=no-store -$(package)_config_opts+=no-unit-test -$(package)_config_opts+=no-weak-ssl-ciphers -$(package)_config_opts+=no-whirlpool -$(package)_config_opts+=no-zlib -$(package)_config_opts+=no-zlib-dynamic -$(package)_config_opts+=$($(package)_cflags) $($(package)_cppflags) -$(package)_config_opts_linux=-fPIC -Wa,--noexecstack -$(package)_config_opts_x86_64_linux=linux-x86_64 -$(package)_config_opts_i686_linux=linux-generic32 -$(package)_config_opts_arm_linux=linux-generic32 -$(package)_config_opts_armv7l_linux=linux-generic32 -$(package)_config_opts_aarch64_linux=linux-generic64 -$(package)_config_opts_mipsel_linux=linux-generic32 -$(package)_config_opts_mips_linux=linux-generic32 -$(package)_config_opts_powerpc_linux=linux-generic32 -$(package)_config_opts_riscv32_linux=linux-generic32 -$(package)_config_opts_riscv64_linux=linux-generic64 -$(package)_config_opts_x86_64_darwin=darwin64-x86_64-cc -$(package)_config_opts_x86_64_mingw32=mingw64 -$(package)_config_opts_i686_mingw32=mingw -endef - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch && \ - sed -i.old "/define DATE/d" util/mkbuildinf.pl && \ - sed -i.old "s|engines apps test|engines|" Makefile.org -endef - -define $(package)_config_cmds - ./Configure $($(package)_config_opts) -endef - -define $(package)_build_cmds - $(MAKE) -j1 build_crypto libcrypto.pc libssl.pc openssl.pc -endef - -define $(package)_stage_cmds - $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw -endef - -define $(package)_postprocess_cmds - rm -rf share bin etc -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 35f8b829db..d8ccb208cc 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,8 +1,11 @@ -packages:=boost openssl libevent +packages:=boost libevent -qt_packages = qrencode zlib +qt_packages = zlib + +qrencode_packages = qrencode qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig +qt_android_packages=qt rapidcheck_packages = rapidcheck diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 21570726ff..d1687883bc 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -9,6 +9,7 @@ $(package)_config_opts=--disable-shared --without-tools --without-tests --disabl $(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_linux=--with-pic +$(package)_config_opts_android=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 0685eb2da2..efa76965d5 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -8,7 +8,7 @@ $(package)_dependencies=zlib $(package)_linux_dependencies=freetype fontconfig libxcb $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch +$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c @@ -25,7 +25,6 @@ $(package)_config_opts_debug = -debug $(package)_config_opts += -bindir $(build_prefix)/bin $(package)_config_opts += -c++std c++11 $(package)_config_opts += -confirm-license -$(package)_config_opts += -dbus-runtime $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-compile-examples $(package)_config_opts += -no-cups @@ -40,12 +39,14 @@ $(package)_config_opts += -no-iconv $(package)_config_opts += -no-kms $(package)_config_opts += -no-linuxfb $(package)_config_opts += -no-libjpeg +$(package)_config_opts += -no-libproxy $(package)_config_opts += -no-libudev $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations $(package)_config_opts += -no-qml-debug +$(package)_config_opts += -no-sctp $(package)_config_opts += -no-securetransport $(package)_config_opts += -no-sql-db2 $(package)_config_opts += -no-sql-ibase @@ -56,12 +57,13 @@ $(package)_config_opts += -no-sql-odbc $(package)_config_opts += -no-sql-psql $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 +$(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker $(package)_config_opts += -no-xinput2 $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -opensource -$(package)_config_opts += -optimized-qmake +$(package)_config_opts += -optimized-tools $(package)_config_opts += -pch $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) @@ -80,9 +82,12 @@ $(package)_config_opts += -no-feature-dial $(package)_config_opts += -no-feature-filesystemwatcher $(package)_config_opts += -no-feature-fontcombobox $(package)_config_opts += -no-feature-ftp +$(package)_config_opts += -no-feature-http $(package)_config_opts += -no-feature-image_heuristic_mask $(package)_config_opts += -no-feature-keysequenceedit $(package)_config_opts += -no-feature-lcdnumber +$(package)_config_opts += -no-feature-networkdiskcache +$(package)_config_opts += -no-feature-networkproxy $(package)_config_opts += -no-feature-pdf $(package)_config_opts += -no-feature-printdialog $(package)_config_opts += -no-feature-printer @@ -90,6 +95,7 @@ $(package)_config_opts += -no-feature-printpreviewdialog $(package)_config_opts += -no-feature-printpreviewwidget $(package)_config_opts += -no-feature-regularexpression $(package)_config_opts += -no-feature-sessionmanager +$(package)_config_opts += -no-feature-socks5 $(package)_config_opts += -no-feature-sql $(package)_config_opts += -no-feature-statemachine $(package)_config_opts += -no-feature-syntaxhighlighter @@ -105,8 +111,11 @@ $(package)_config_opts += -no-feature-vnc $(package)_config_opts += -no-feature-wizard $(package)_config_opts += -no-feature-xml +$(package)_config_opts_darwin = -no-dbus +$(package)_config_opts_darwin += -no-opengl + ifneq ($(build_os),darwin) -$(package)_config_opts_darwin = -xplatform macx-clang-linux +$(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSION) $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" @@ -122,12 +131,41 @@ $(package)_config_opts_linux += -no-feature-xlib $(package)_config_opts_linux += -system-freetype $(package)_config_opts_linux += -fontconfig $(package)_config_opts_linux += -no-opengl +$(package)_config_opts_linux += -dbus-runtime $(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_i686_linux = -xplatform linux-g++-32 $(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 $(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ +$(package)_config_opts_powerpc64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ +$(package)_config_opts_powerpc64le_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_riscv64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_mingw32 = -no-opengl -xplatform win32-g++ -device-option CROSS_COMPILE="$(host)-" +$(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ + +$(package)_config_opts_mingw32 = -no-opengl +$(package)_config_opts_mingw32 += -no-dbus +$(package)_config_opts_mingw32 += -xplatform win32-g++ +$(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" + +$(package)_config_opts_android = -xplatform android-clang +$(package)_config_opts_android += -android-sdk $(ANDROID_SDK) +$(package)_config_opts_android += -android-ndk $(ANDROID_NDK) +$(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LEVEL) +$(package)_config_opts_android += -device-option CROSS_COMPILE="$(host)-" +$(package)_config_opts_android += -egl +$(package)_config_opts_android += -qpa xcb +$(package)_config_opts_android += -no-eglfs +$(package)_config_opts_android += -no-dbus +$(package)_config_opts_android += -opengl es2 +$(package)_config_opts_android += -qt-freetype +$(package)_config_opts_android += -no-fontconfig +$(package)_config_opts_android += -L $(host_prefix)/lib +$(package)_config_opts_android += -I $(host_prefix)/include + +$(package)_config_opts_aarch64_android += -android-arch arm64-v8a +$(package)_config_opts_armv7a_android += -android-arch armeabi-v7a +$(package)_config_opts_x86_64_android += -android-arch x86_64 +$(package)_config_opts_i686_android += -android-arch i686 + $(package)_build_env = QT_RCC_TEST=1 $(package)_build_env += QT_RCC_SOURCE_DATE_OVERRIDE=1 endef @@ -170,6 +208,8 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch &&\ patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch &&\ patch -p1 -i $($(package)_patch_dir)/xkb-default.patch &&\ + patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch &&\ + patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch &&\ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ @@ -180,6 +220,8 @@ define $(package)_preprocess_cmds sed -i.old "s|QMAKE_CFLAGS += |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_CXXFLAGS += |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \ + sed -i.old "s|QMAKE_CC = clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ + sed -i.old "s|QMAKE_CXX = clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf endef diff --git a/depends/packages/xcb_proto.mk b/depends/packages/xcb_proto.mk index 85d01ecd2f..01203a0718 100644 --- a/depends/packages/xcb_proto.mk +++ b/depends/packages/xcb_proto.mk @@ -4,11 +4,6 @@ $(package)_download_path=https://xcb.freedesktop.org/dist $(package)_file_name=xcb-proto-$($(package)_version).tar.bz2 $(package)_sha256_hash=7ef40ddd855b750bc597d2a435da21e55e502a0fefa85b274f2c922800baaf05 -define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-option-checking - $(package)_config_opts_linux=--with-pic -endef - define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index ce85c437bb..6f35ede248 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -11,6 +11,7 @@ define $(package)_set_vars $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic + $(package)_config_opts_android=--with-pic $(package)_cxxflags=-std=c++11 endef diff --git a/depends/packages/zlib.mk b/depends/packages/zlib.mk index 168f85e65e..acb02020a8 100644 --- a/depends/packages/zlib.mk +++ b/depends/packages/zlib.mk @@ -11,6 +11,7 @@ $(package)_config_opts+=RANLIB="$($(package)_ranlib)" $(package)_config_opts+=AR="$($(package)_ar)" $(package)_config_opts_darwin+=AR="$($(package)_libtool)" $(package)_config_opts_darwin+=ARFLAGS="-o" +$(package)_config_opts_android+=CHOST=$(host) endef # zlib has its own custom configure script that takes in options like CC, diff --git a/depends/patches/openssl/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch b/depends/patches/openssl/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch deleted file mode 100644 index 003099bdc2..0000000000 --- a/depends/patches/openssl/0001-Add-OpenSSL-termios-fix-for-musl-libc.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/crypto/ui/ui_openssl.c b/crypto/ui/ui_openssl.c -index a38c758..d99edc2 100644 ---- a/crypto/ui/ui_openssl.c -+++ b/crypto/ui/ui_openssl.c -@@ -190,9 +190,9 @@ - # undef SGTTY - #endif - --#if defined(linux) && !defined(TERMIO) --# undef TERMIOS --# define TERMIO -+#if defined(linux) -+# define TERMIOS -+# undef TERMIO - # undef SGTTY - #endif - diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch new file mode 100644 index 0000000000..2f6ff00f40 --- /dev/null +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -0,0 +1,18 @@ +--- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp ++++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp +@@ -890,6 +890,14 @@ + __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); + return -1; + } ++ ++ const jint ret = QT_PREPEND_NAMESPACE(QtAndroidPrivate::initJNI(vm, env)); ++ if (ret != 0) ++ { ++ __android_log_print(ANDROID_LOG_FATAL, "Qt", "initJNI failed"); ++ return ret; ++ } ++ + QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); + + m_javaVM = vm; + diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch new file mode 100644 index 0000000000..13bfff9776 --- /dev/null +++ b/depends/patches/qt/fix_android_qmake_conf.patch @@ -0,0 +1,20 @@ +--- old/qtbase/mkspecs/android-clang/qmake.conf ++++ new/qtbase/mkspecs/android-clang/qmake.conf +@@ -30,7 +30,7 @@ + QMAKE_CFLAGS += -target mips64el-none-linux-android + + QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH +-QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a ++QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -nostdlib++ + QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ + -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ + -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ +@@ -40,7 +40,7 @@ + ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH + + ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so +-ANDROID_CXX_STL_LIBS = -lc++ ++ANDROID_CXX_STL_LIBS = -lc++_shared + + QMAKE_ARM_CFLAGS_RELEASE = -Oz + QMAKE_ARM_CFLAGS_RELEASE_WITH_DEBUGINFO = -g -Oz diff --git a/doc/README.md b/doc/README.md index d3017de2ab..4dd21c4620 100644 --- a/doc/README.md +++ b/doc/README.md @@ -54,7 +54,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Productivity Notes](productivity.md) - [Release Notes](release-notes.md) - [Release Process](release-process.md) -- [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) +- [Source Code Documentation (External Link)](https://doxygen.bitcoincore.org/) - [Translation Process](translation_process.md) - [Translation Strings Policy](translation_strings_policy.md) - [JSON-RPC Interface](JSON-RPC-interface.md) @@ -84,5 +84,3 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th License --------------------- Distributed under the [MIT software license](/COPYING). -This product includes software developed by the OpenSSL Project for use in the [OpenSSL Toolkit](https://www.openssl.org/). This product includes -cryptographic software written by Eric Young ([eay@cryptsoft.com](mailto:eay@cryptsoft.com)), and UPnP software written by Thomas Bernard. diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 3b8b0db162..a63391e01a 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -50,7 +50,7 @@ Given a height: returns hash of block in best-block-chain at height provided. Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name as defined in BIP70 (main, test, regtest) +* chain : (string) current network name (main, test, regtest) * blocks : (numeric) the current number of blocks processed in the server * headers : (numeric) the current number of headers we have validated * bestblockhash : (string) the hash of the currently best block diff --git a/doc/bips.md b/doc/bips.md index 45562cec62..b96862297f 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -19,7 +19,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.19.0**): * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 68`](https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki): Sequence locks have been implemented as of **v0.12.1** ([PR #7184](https://github.com/bitcoin/bitcoin/pull/7184)), and have been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). -* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), and is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). +* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): + Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). + Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), + and it is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). + It has been removed as of **v0.20.0** ([PR 17165](https://github.com/bitcoin/bitcoin/pull/17165)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index d22b6e8383..4831623504 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -11,7 +11,7 @@ This guide does not contain instructions for building the GUI. You will need the following dependencies, which can be installed as root via pkg: ```shell -pkg install autoconf automake boost-libs git gmake libevent libtool openssl pkgconf +pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf git clone https://github.com/bitcoin/bitcoin.git ``` diff --git a/doc/build-osx.md b/doc/build-osx.md index 9942449bf6..bf655538c7 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -19,7 +19,7 @@ Then install [Homebrew](https://brew.sh). ## Dependencies ```shell -brew install automake berkeley-db4 libtool boost miniupnpc openssl pkg-config python qt libevent qrencode +brew install automake berkeley-db4 libtool boost miniupnpc pkg-config python qt libevent qrencode ``` See [dependencies.md](dependencies.md) for a complete overview. @@ -110,7 +110,7 @@ tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log ``` ## Notes -* Tested on OS X 10.10 Yosemite through macOS 10.14 Mojave on 64-bit Intel +* Tested on OS X 10.12 Sierra through macOS 10.15 Catalina on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714). diff --git a/doc/build-unix.md b/doc/build-unix.md index d048bdeff5..e799e709fa 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -33,7 +33,6 @@ These dependencies are required: Library | Purpose | Description ------------|------------------|---------------------- - libssl | Crypto | Random Number Generation, Elliptic Curve Cryptography libboost | Utility | Library for threading, data structures, etc libevent | Networking | OS independent asynchronous networking @@ -81,7 +80,7 @@ Build requirements: Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: - sudo apt-get install libssl-dev libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev + sudo apt-get install libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev BerkeleyDB is required for the wallet. @@ -127,7 +126,7 @@ built by default. Build requirements: - sudo dnf install gcc-c++ libtool make autoconf automake openssl-devel libevent-devel boost-devel libdb4-devel libdb4-cxx-devel python3 + sudo dnf install gcc-c++ libtool make autoconf automake libevent-devel boost-devel libdb4-devel libdb4-cxx-devel python3 Optional (see `--with-miniupnpc` and `--enable-upnp-default`): diff --git a/doc/build-windows.md b/doc/build-windows.md index f8095f6a65..bbff638b90 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -8,18 +8,18 @@ The options known to work for building Bitcoin Core on Windows are: * On Linux, using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Bionic 18.04 is required and is the platform used to build the Bitcoin Core Windows release binaries. * On Windows, using [Windows -Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about) and the Mingw-w64 cross compiler tool chain. +Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. +* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). Other options which may work, but which have not been extensively tested are (please contribute instructions): -* On Windows, using a POSIX compatibility layer application such as [cygwin](http://www.cygwin.com/) or [msys2](http://www.msys2.org/). -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). +* On Windows, using a POSIX compatibility layer application such as [cygwin](https://www.cygwin.com/) or [msys2](https://www.msys2.org/). Installing Windows Subsystem for Linux --------------------------------------- With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about). This +Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu-based environment. Within this environment you can cross compile for Windows without the need for a separate Linux VM or server. Note that while WSL can be installed with @@ -28,7 +28,7 @@ tested with Ubuntu. This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. In addition, it is available [only for 64-bit versions of -Windows](https://msdn.microsoft.com/en-us/commandline/wsl/install_guide). +Windows](https://docs.microsoft.com/windows/wsl/install-win10). Full instructions to install WSL are available on the above link. To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: diff --git a/doc/dependencies.md b/doc/dependencies.md index dc88626761..51a2240107 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -10,18 +10,17 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Clang | | [3.3+](https://releases.llvm.org/download.html) (C++11 support) | | | | | Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | | +| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | -| HarfBuzz-NG | | | | | | -| libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | +| HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | +| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | -| MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | -| OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | +| MiniUPnPc | [2.0.20180203](https://miniupnp.tuxfamily.org/files) | | No | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | +| Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | diff --git a/doc/descriptors.md b/doc/descriptors.md index a98f43737e..e31665b129 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -10,6 +10,13 @@ Supporting RPCs are: - `deriveaddresses` takes as input a descriptor and computes the corresponding addresses. - `listunspent` outputs a specialized descriptor for the reported unspent outputs. +- `getaddressinfo` outputs a descriptor for solvable addresses (since v0.18). +- `importmulti` takes as input descriptors to import into the wallet + (since v0.18). +- `generatetodescriptor` takes as input a descriptor and generates coins to it + (`regtest` only, since v0.19). +- `utxoupdatepsbt` takes as input descriptors to add information to the psbt + (since v0.19). This document describes the language. For the specifics on usage, see the RPC documentation for the functions mentioned above. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 9cf4b4b075..3ef35cfcfa 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -9,15 +9,17 @@ Developer Notes - [Coding Style (C++)](#coding-style-c) - [Coding Style (Python)](#coding-style-python) - [Coding Style (Doxygen-compatible comments)](#coding-style-doxygen-compatible-comments) + - [Generating Documentation](#generating-documentation) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - - [debug.log](#debuglog) + - [`debug.log`](#debuglog) - [Testnet and Regtest modes](#testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) - [Performance profiling with perf](#performance-profiling-with-perf) + - [Sanitizers](#sanitizers) - [Locking/mutex usage notes](#lockingmutex-usage-notes) - [Threads](#threads) - [Ignoring IDE/editor files](#ignoring-ideeditor-files) @@ -34,7 +36,11 @@ Developer Notes - [Source code organization](#source-code-organization) - [GUI](#gui) - [Subtrees](#subtrees) + - [Upgrading LevelDB](#upgrading-leveldb) + - [File Descriptor Counts](#file-descriptor-counts) + - [Consensus Compatibility](#consensus-compatibility) - [Scripted diffs](#scripted-diffs) + - [Suggestions and examples](#suggestions-and-examples) - [Release notes](#release-notes) - [RPC interface guidelines](#rpc-interface-guidelines) @@ -62,7 +68,7 @@ tool to clean up patches automatically before submission. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - - No extra spaces inside parenthesis; don't do ( this ). + - No extra spaces inside parenthesis; don't do `( this )`. - No space after function names; one space after `if`, `for` and `while`. - If an `if` only has a single-statement `then`-clause, it can appear on the same line as the `if`, without braces. In every other case, @@ -76,7 +82,7 @@ code. separate words (snake_case). - Class member variables have a `m_` prefix. - Global variables have a `g_` prefix. - - Constant names are all uppercase, and use `_` to separate words. + - Compile-time constant names are all uppercase, and use `_` to separate words. - Class names, function names, and method names are UpperCamelCase (PascalCase). Do not prefix class names with `C`. - Test suite naming convention: The Boost test suite in file @@ -136,12 +142,17 @@ For example, to describe a function use: ```c++ /** - * ... text ... - * @param[in] arg1 A description - * @param[in] arg2 Another argument description - * @pre Precondition for function... + * ... Description ... + * + * @param[in] arg1 input description... + * @param[in] arg2 input description... + * @param[out] arg3 output description... + * @return Return cases... + * @throws Error type and cases... + * @pre Pre-condition for function... + * @post Post-condition for function... */ -bool function(int arg1, const char *arg2) +bool function(int arg1, const char *arg2, std::string& arg3) ``` A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. @@ -156,44 +167,73 @@ To describe a class, use the same construct above the class definition: * @see GetWarnings() */ class CAlert -{ ``` To describe a member or variable use: ```c++ -int var; //!< Detailed description after the member +//! Description before the member +int var; ``` or ```c++ -//! Description before the member -int var; +int var; //!< Description after the member ``` Also OK: ```c++ /// -/// ... text ... +/// ... Description ... /// bool function2(int arg1, const char *arg2) ``` -Not OK (used plenty in the current source, but not picked up): +Not picked up by Doxygen: ```c++ // -// ... text ... +// ... Description ... // ``` +Also not picked up by Doxygen: +```c++ +/* + * ... Description ... + */ +``` + A full list of comment syntaxes picked up by Doxygen can be found at http://www.doxygen.nl/manual/docblocks.html, but the above styles are favored. -Documentation can be generated with `make docs` and cleaned up with `make clean-docs`. The resulting files are located in `doc/doxygen/html`; open `index.html` to view the homepage. +Recommendations: -Before running `make docs`, you will need to install dependencies `doxygen` and `dot`. For example, on macOS via Homebrew: -``` -brew install graphviz doxygen -``` +- Avoiding duplicating type and input/output information in function + descriptions. + +- Use backticks (``) to refer to `argument` names in function and + parameter descriptions. + +- Backticks aren't required when referring to functions Doxygen already knows + about; it will build hyperlinks for these automatically. See + http://www.doxygen.nl/manual/autolink.html for complete info. + +- Avoid linking to external documentation; links can break. + +- Javadoc and all valid Doxygen comments are stripped from Doxygen source code + previews (`STRIP_CODE_COMMENTS = YES` in [Doxyfile.in](doc/Doxyfile.in)). If + you want a comment to be preserved, it must instead use `//` or `/* */`. + +### Generating Documentation + +The documentation can be generated with `make docs` and cleaned up with `make +clean-docs`. The resulting files are located in `doc/doxygen/html`; open +`index.html` in that directory to view the homepage. + +Before running `make docs`, you'll need to install these dependencies: + +Linux: `sudo apt install doxygen graphviz` + +MacOS: `brew install doxygen graphviz` Development tips and tricks --------------------------- @@ -207,15 +247,15 @@ produce better debugging builds. Run configure with the `--enable-gprof` option, then make. -### debug.log +### `debug.log` -If the code is behaving strangely, take a look in the debug.log file in the data directory; +If the code is behaving strangely, take a look in the `debug.log` file in the data directory; error and debugging messages are written there. The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn -on all categories (and give you a very large debug.log file). +on all categories (and give you a very large `debug.log` file). -The Qt code routes `qDebug()` output to debug.log under category "qt": run with `-debug=qt` +The Qt code routes `qDebug()` output to `debug.log` under category "qt": run with `-debug=qt` to see it. ### Testnet and Regtest modes @@ -233,7 +273,7 @@ Bitcoin Core is a multi-threaded application, and deadlocks or other multi-threading bugs can be very difficult to track down. The `--enable-debug` configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held and adds warnings to the -debug.log file if inconsistencies are detected. +`debug.log` file if inconsistencies are detected. ### Valgrind suppressions file @@ -275,8 +315,7 @@ the functional test framework. Perf can observe a running process and sample (at some frequency) where its execution is. Perf installation is contingent on which kernel version you're running; see -[this StackExchange -thread](https://askubuntu.com/questions/50145/how-to-install-perf-monitoring-tool) +[this thread](https://askubuntu.com/questions/50145/how-to-install-perf-monitoring-tool) for specific instructions. Certain kernel parameters may need to be set for perf to be able to inspect the @@ -311,7 +350,7 @@ or using a graphical tool like [Hotspot](https://github.com/KDAB/hotspot). See the functional test documentation for how to invoke perf within tests. -**Sanitizers** +### Sanitizers Bitcoin Core can be compiled with various "sanitizers" enabled, which add instrumentation for issues regarding things like memory safety, thread race @@ -372,7 +411,7 @@ Deadlocks due to inconsistent lock ordering (thread 1 locks `cs_main` and then `cs_wallet`, while thread 2 locks them in the opposite order: result, deadlock as each waits for the other to release its lock) are a problem. Compile with `-DDEBUG_LOCKORDER` (or use `--enable-debug`) to get lock order inconsistencies -reported in the debug.log file. +reported in the `debug.log` file. Re-architecting the core code so there are better-defined interfaces between the various components is a goal, with any necessary locking @@ -384,9 +423,7 @@ Threads - ThreadScriptCheck : Verifies block scripts. -- ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat. - -- StartNode : Starts other threads. +- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`. - ThreadDNSAddressSeed : Loads addresses of peers from the DNS. @@ -400,7 +437,7 @@ Threads - ThreadMessageHandler : Higher-level message handling (sending and receiving). -- DumpAddresses : Dumps IP addresses of nodes to peers.dat. +- DumpAddresses : Dumps IP addresses of nodes to `peers.dat`. - ThreadRPCServer : Remote procedure call handler, listens on port 8332 for connections and services them. @@ -466,11 +503,6 @@ Wallet - Make sure that no crashes happen with run-time option `-disablewallet`. - - *Rationale*: In RPC code that conditionally uses the wallet (such as - `validateaddress`), it is easy to forget that global pointer `pwalletMain` - can be nullptr. See `test/functional/disablewallet.py` for functional tests - exercising the API with `-disablewallet`. - - Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set. - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB. @@ -545,11 +577,10 @@ class A } ``` -- By default, declare single-argument constructors `explicit`. +- By default, declare constructors `explicit`. - - *Rationale*: This is a precaution to avoid unintended conversions that might - arise when single-argument constructors are used as implicit conversion - functions. + - *Rationale*: This is a precaution to avoid unintended + [conversions](https://en.cppreference.com/w/cpp/language/converting_constructor). - Use explicitly signed or unsigned `char`s, or even better `uint8_t` and `int8_t`. Do not use bare `char` unless it is to pass to a third-party API. @@ -640,6 +671,28 @@ Strings and formatting - *Rationale*: Bitcoin Core uses tinyformat, which is type safe. Leave them out to avoid confusion. +- Use `.c_str()` sparingly. Its only valid use is to pass C++ strings to C functions that take NULL-terminated + strings. + + - Do not use it when passing a sized array (so along with `.size()`). Use `.data()` instead to get a pointer + to the raw data. + + - *Rationale*: Although this is guaranteed to be safe starting with C++11, `.data()` communicates the intent better. + + - Do not use it when passing strings to `tfm::format`, `strprintf`, `LogPrint[f]`. + + - *Rationale*: This is redundant. Tinyformat handles strings. + + - Do not use it to convert to `QString`. Use `QString::fromStdString()`. + + - *Rationale*: Qt has built-in functionality for converting their string + type from/to C++. No need to roll your own. + + - In cases where do you call `.c_str()`, you might want to additionally check that the string does not contain embedded '\0' characters, because + it will (necessarily) truncate the string. This might be used to hide parts of the string from logging or to circumvent + checks. If a use of strings is sensitive to this, take care to check the string for embedded NULL characters first + and reject it if there are any (see `ParsePrechecks` in `strencodings.cpp` for an example). + Shadowing -------------- @@ -805,7 +858,7 @@ Current subtrees include: - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when merging upstream changes to the LevelDB subtree. -- src/libsecp256k1 +- src/secp256k1 - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintained by Core contributors. - src/crypto/ctaes @@ -867,7 +920,7 @@ Scripted diffs For reformatting and refactoring commits where the changes can be easily automated using a bash script, we use scripted-diff commits. The bash script is included in the commit message and our Travis CI job checks that the result of the script is identical to the commit. This aids reviewers since they can verify that the script -does exactly what it's supposed to do. It is also helpful for rebasing (since the same script can just be re-run +does exactly what it is supposed to do. It is also helpful for rebasing (since the same script can just be re-run on the new master commit). To create a scripted-diff: @@ -888,7 +941,35 @@ For development, it might be more convenient to verify all scripted-diffs in a r test/lint/commit-script-check.sh origin/master..HEAD ``` -Commit [`bb81e173`](https://github.com/bitcoin/bitcoin/commit/bb81e173) is an example of a scripted-diff. +### Suggestions and examples + +If you need to replace in multiple files, prefer `git ls-files` to `find` or globbing, and `git grep` to `grep`, to +avoid changing files that are not under version control. + +For efficient replacement scripts, reduce the selection to the files that potentially need to be modified, so for +example, instead of a blanket `git ls-files src | xargs sed -i s/apple/orange/`, use +`git grep -l apple src | xargs sed -i s/apple/orange/`. + +Also, it is good to keep the selection of files as specific as possible — for example, replace only in directories where +you expect replacements — because it reduces the risk that a rebase of your commit by re-running the script will +introduce accidental changes. + +Some good examples of scripted-diff: + +- [scripted-diff: Rename InitInterfaces to NodeContext](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) +uses an elegant script to replace occurrences of multiple terms in all source files. + +- [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) +replaces specific terms in a list of specific source files. + +- [scripted-diff: Replace fprintf with tfm::format](https://github.com/bitcoin/bitcoin/commit/fac03ec43a15ad547161e37e53ea82482cc508f9) +does a global replacement but excludes certain directories. + +To find all previous uses of scripted diffs in the repository, do: + +``` +git log --grep="-BEGIN VERIFY SCRIPT-" +``` Release notes ------------- @@ -971,7 +1052,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: - A RPC method must either be a wallet method or a non-wallet method. Do not introduce new methods that differ in behavior based on the presence of a wallet. - - *Rationale*: as well as complicating the implementation and interfering + - *Rationale*: As well as complicating the implementation and interfering with the introduction of multi-wallet, wallet and non-wallet code should be separated to avoid introducing circular dependencies between code units. @@ -998,8 +1079,18 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: RPC methods registered with the same function pointer will be considered aliases and only the first method name will show up in the - `help` rpc command list. + `help` RPC command list. - *Exception*: Using RPC method aliases may be appropriate in cases where a new RPC is replacing a deprecated RPC, to avoid both RPCs confusingly showing up in the command list. + +- Use *invalid* bech32 addresses for `RPCExamples` help documentation. + + - *Rationale*: Prevent accidental transactions by users and encourage the use + of bech32 addresses by default. + +- Use the `UNIX_EPOCH_TIME` constant when describing UNIX epoch time or + timestamps in the documentation. + + - *Rationale*: User-facing consistency. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 3dc6be8b86..c34ca4cb59 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -7,11 +7,8 @@ describe how to use it with AFL and libFuzzer. ## Preparing fuzzing -AFL needs an input directory with examples, and an output directory where it -will place examples that it found. These can be anywhere in the file system, -we'll define environment variables to make it easy to reference them. - -libFuzzer will use the input directory as output directory. +The fuzzer needs some inputs to work on, but the inputs or seeds can be used +interchangeably between libFuzzer and AFL. Extract the example seeds (or other starting inputs) into the inputs directory before starting fuzzing. @@ -21,13 +18,19 @@ git clone https://github.com/bitcoin-core/qa-assets export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus ``` -Only for AFL: +AFL needs an input directory with examples, and an output directory where it +will place examples that it found. These can be anywhere in the file system, +we'll define environment variables to make it easy to reference them. + +So, only for AFL you need to configure the outputs path: ``` mkdir outputs export AFLOUT=$PWD/outputs ``` +libFuzzer will use the input directory as output directory. + ## AFL ### Building AFL @@ -41,6 +44,9 @@ make export AFLPATH=$PWD ``` +For macOS you may need to ignore x86 compilation checks when running `make`: +`AFL_NO_X86=1 make`. + ### Instrumentation To build Bitcoin Core using AFL instrumentation (this assumes that the @@ -48,9 +54,15 @@ To build Bitcoin Core using AFL instrumentation (this assumes that the ``` ./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ export AFL_HARDEN=1 -cd src/ make ``` + +If you are using clang you will need to substitute `afl-gcc` with `afl-clang` +and `afl-g++` with `afl-clang++`, so the first line above becomes: +``` +./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-clang CXX=${AFLPATH}/afl-clang++ +``` + We disable ccache because we don't want to pollute the ccache with instrumented objects, and similarly don't want to use non-instrumented cached objects linked in. @@ -60,38 +72,67 @@ The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting binary will be instrumented in such a way that the AFL features "persistent mode" and "deferred forkserver" can be used. See -https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. +https://github.com/google/AFL/tree/master/llvm_mode for details. ### Fuzzing To start the actual fuzzing use: ``` -export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target +export FUZZ_TARGET=bech32 # Pick a fuzz_target mkdir ${AFLOUT}/${FUZZ_TARGET} -$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET} +$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} ``` You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. +On macOS you may need to set `AFL_NO_FORKSRV=1` to get the target to run. +``` +export FUZZ_TARGET=bech32 # Pick a fuzz_target +mkdir ${AFLOUT}/${FUZZ_TARGET} +AFL_NO_FORKSRV=1 $AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} +``` + ## libFuzzer -A recent version of `clang`, the address sanitizer and libFuzzer is needed (all -found in the `compiler-rt` runtime libraries package). +A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and +libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). To build all fuzz targets with libFuzzer, run ``` -./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ +./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ make ``` -The fuzzer needs some inputs to work on, but the inputs or seeds can be used -interchangeably between libFuzzer and AFL. - See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer instrumented executable. -Alternatively run the script in `./test/fuzz/test_runner.py` and provide it -with the `${DIR_FUZZ_IN}` created earlier. +Alternatively, you can run the script through the fuzzing test harness (only +libFuzzer supported so far). You need to pass it the inputs directory and +the specific test target you want to run. + +``` +./test/fuzz/test_runner.py ${DIR_FUZZ_IN} bech32 +``` + +### macOS hints for libFuzzer + +The default clang/llvm version supplied by Apple on macOS does not include +fuzzing libraries, so macOS users will need to install a full version, for +example using `brew install llvm`. + +Should you run into problems with the address sanitizer, it is possible you +may need to run `./configure` with `--disable-asm` to avoid errors +with certain assembly code from Bitcoin Core's code. See [developer notes on sanitizers](https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#sanitizers) +for more information. + +You may also need to take care of giving the correct path for clang and +clang++, like `CC=/path/to/clang CXX=/path/to/clang++` if the non-systems +clang does not come first in your path. + +Full configure that was tested on macOS Catalina with `brew` installed `llvm`: +``` +./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm +``` diff --git a/doc/man/bitcoin-cli.1 b/doc/man/bitcoin-cli.1 index 95c1d24dff..129651d8e9 100644 --- a/doc/man/bitcoin-cli.1 +++ b/doc/man/bitcoin-cli.1 @@ -113,7 +113,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoin-qt.1 b/doc/man/bitcoin-qt.1 index 1957fb736e..f68be21e8d 100644 --- a/doc/man/bitcoin-qt.1 +++ b/doc/man/bitcoin-qt.1 @@ -608,7 +608,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoin-tx.1 b/doc/man/bitcoin-tx.1 index 6b6071d9b7..b4c7698896 100644 --- a/doc/man/bitcoin-tx.1 +++ b/doc/man/bitcoin-tx.1 @@ -114,7 +114,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoin-wallet.1 b/doc/man/bitcoin-wallet.1 index 1cb8cdebcd..aadea09a2b 100644 --- a/doc/man/bitcoin-wallet.1 +++ b/doc/man/bitcoin-wallet.1 @@ -61,7 +61,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoind.1 b/doc/man/bitcoind.1 index b0aff99ca2..211ba10285 100644 --- a/doc/man/bitcoind.1 +++ b/doc/man/bitcoind.1 @@ -581,7 +581,3 @@ The source code is available from <https://github.com/bitcoin/bitcoin>. This is experimental software. Distributed under the MIT software license, see the accompanying file COPYING or <https://opensource.org/licenses/MIT> - -This product includes software developed by the OpenSSL Project for use in the -OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by -Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 8d8ccdfedc..b0faf0825a 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -41,7 +41,7 @@ threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a By default, since glibc `2.10`, the C library will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: ```bash -#!/bin/bash +#!/usr/bin/env bash export MALLOC_ARENA_MAX=1 bitcoind ``` diff --git a/doc/release-notes-15954.md b/doc/release-notes-15954.md new file mode 100644 index 0000000000..f4d2c5688c --- /dev/null +++ b/doc/release-notes-15954.md @@ -0,0 +1,4 @@ +Configuration option changes +----------------------------- + +Importing blocks upon startup via the `bootstrap.dat` file no longer occurs by default. The file must now be specified with `-loadblock=<file>`. diff --git a/doc/release-notes-17410.md b/doc/release-notes-17410.md new file mode 100644 index 0000000000..08ed353889 --- /dev/null +++ b/doc/release-notes-17410.md @@ -0,0 +1,5 @@ +Command-line options +-------------------- + +- The `-debug=db` logging category has been renamed to `-debug=walletdb`, to distinguish it from `coindb`. + `-debug=db` has been deprecated and will be removed in the next major release. diff --git a/doc/release-notes-17437.md b/doc/release-notes-17437.md new file mode 100644 index 0000000000..3edfd00a38 --- /dev/null +++ b/doc/release-notes-17437.md @@ -0,0 +1,5 @@ +Low-level RPC Changes +=== + +- The RPC gettransaction, listtransactions and listsinceblock responses now also +includes the height of the block that contains the wallet transaction, if any. diff --git a/doc/release-notes-17578.md b/doc/release-notes-17578.md new file mode 100644 index 0000000000..664d17fd78 --- /dev/null +++ b/doc/release-notes-17578.md @@ -0,0 +1,13 @@ +Deprecated or removed RPCs +-------------------------- + +- RPC `getaddressinfo` changes: + + - the `label` field has been deprecated in favor of the `labels` field and + will be removed in 0.21. It can be re-enabled in the interim by launching + with `-deprecatedrpc=label`. + + - the `labels` behavior of returning an array of JSON objects containing name + and purpose key/value pairs has been deprecated in favor of an array of + label names and will be removed in 0.21. The previous behavior can be + re-enabled in the interim by launching with `-deprecatedrpc=labelspurpose`. diff --git a/doc/release-notes.md b/doc/release-notes.md index ea82962e75..eec1ef9c46 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -46,16 +46,15 @@ Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems using -the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended +the Linux kernel, macOS 10.12+, and Windows 7 and newer. It is not recommended to use Bitcoin Core on unsupported systems. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. -From Bitcoin Core 0.17.0 onwards, macOS versions earlier than 10.10 are no -longer supported, as Bitcoin Core is now built using Qt 5.9.x which requires -macOS 10.10+. Additionally, Bitcoin Core does not yet change appearance when -macOS "dark mode" is activated. +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. In addition to previously supported CPU platforms, this release's pre-compiled distribution provides binaries for the RISC-V platform. @@ -63,12 +62,25 @@ distribution provides binaries for the RISC-V platform. Notable changes =============== +Build System +------------ + +- OpenSSL is no longer used by Bitcoin Core. The last usage of the library +was removed in #17265. + +- glibc 2.17 or greater is now required to run the release binaries. This +retains compatibility with RHEL 7, CentOS 7, Debian 8 and Ubuntu 14.04 LTS. +Further details can be found in #17538. + New RPCs -------- New settings ------------ +- RPC Whitelist system. It can give certain RPC users permissions to only some RPC calls. +It can be set with two command line arguments (`rpcwhitelist` and `rpcwhitelistdefault`). (#12763) + Updated settings ---------------- @@ -81,10 +93,13 @@ Low-level Changes section below. GUI changes ----------- +- The "Start Bitcoin Core on system login" option has been removed on macOS. + Wallet ------ - The wallet now by default uses bech32 addresses when using RPC, and creates native segwit change outputs. +- The way that output trust was computed has been fixed in #16766, which impacts confirmed/unconfirmed balance status and coin selection. Low-level changes ================= @@ -92,6 +107,10 @@ Low-level changes Tests ----- +- It is now an error to use an unqualified `walletdir=path` setting in the config file if running on testnet or regtest + networks. The setting now needs to be qualified as `chain.walletdir=path` or placed in the appropriate `[chain]` + section. (#17447) + - `-fallbackfee` was 0 (disabled) by default for the main chain, but 0.0002 by default for the test chains. Now it is 0 by default for all chains. Testnet and regtest users will have to add `fallbackfee=0.0002` to their configuration if they weren't setting it and they want it to keep working like before. (#16524) diff --git a/doc/release-notes/release-notes-0.19.0.1.md b/doc/release-notes/release-notes-0.19.0.1.md new file mode 100644 index 0000000000..70b8ec589f --- /dev/null +++ b/doc/release-notes/release-notes-0.19.0.1.md @@ -0,0 +1,1089 @@ +Bitcoin Core version 0.19.0.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.19.0.1/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems using +the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended +to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is +built using Qt 5.9.x, which doesn't support versions of macOS older than +10.10. Additionally, Bitcoin Core does not yet change appearance when +macOS "dark mode" is activated. + +Users running macOS Catalina may need to "right-click" and then choose "Open" +to open the Bitcoin Core .dmg. This is due to new signing requirements +imposed by Apple, which the Bitcoin Core project does not yet adhere too. + +Notable changes +=============== + +New user documentation +---------------------- + +- [Reduce memory](https://github.com/bitcoin/bitcoin/blob/master/doc/reduce-memory.md) + suggests configuration tweaks for running Bitcoin Core on systems with + limited memory. (#16339) + +New RPCs +-------- + +- `getbalances` returns an object with all balances (`mine`, + `untrusted_pending` and `immature`). Please refer to the RPC help of + `getbalances` for details. The new RPC is intended to replace + `getbalance`, `getunconfirmedbalance`, and the balance fields in + `getwalletinfo`. These old calls and fields may be removed in a + future version. (#15930, #16239) + +- `setwalletflag` sets and unsets wallet flags that enable or disable + features specific to that existing wallet, such as the new + `avoid_reuse` feature documented elsewhere in these release notes. + (#13756) + +- `getblockfilter` gets the BIP158 filter for the specified block. This + RPC is only enabled if block filters have been created using the + `-blockfilterindex` configuration option. (#14121) + +New settings +------------ + +- `-blockfilterindex` enables the creation of BIP158 block filters for + the entire blockchain. Filters will be created in the background and + currently use about 4 GiB of space. Note: this version of Bitcoin + Core does not serve block filters over the P2P network, although the + local user may obtain block filters using the `getblockfilter` RPC. + (#14121) + +Updated settings +---------------- + +- `whitebind` and `whitelist` now accept a list of permissions to + provide peers connecting using the indicated interfaces or IP + addresses. If no permissions are specified with an address or CIDR + network, the implicit default permissions are the same as previous + releases. See the `bitcoind -help` output for these two options for + details about the available permissions. (#16248) + +- Users setting custom `dbcache` values can increase their setting slightly + without using any more real memory. Recent changes reduced the memory use + by about 9% and made chainstate accounting more accurate (it was underestimating + the use of memory before). For example, if you set a value of "450" before, you + may now set a value of "500" to use about the same real amount of memory. (#16957) + + +Updated RPCs +------------ + +Note: some low-level RPC changes mainly useful for testing are described in the +Low-level Changes section below. + +- `sendmany` no longer has a `minconf` argument. This argument was not + well-specified and would lead to RPC errors even when the wallet's + coin selection succeeded. Users who want to influence coin selection + can use the existing `-spendzeroconfchange`, `-limitancestorcount`, + `-limitdescendantcount` and `-walletrejectlongchains` configuration + arguments. (#15596) + +- `getbalance` and `sendtoaddress`, plus the new RPCs `getbalances` and + `createwallet`, now accept an "avoid_reuse" parameter that controls + whether already used addresses should be included in the operation. + Additionally, `sendtoaddress` will avoid partial spends when + `avoid_reuse` is enabled even if this feature is not already enabled + via the `-avoidpartialspends` command line flag because not doing so + would risk using up the "wrong" UTXO for an address reuse case. + (#13756) + +- RPCs which have an `include_watchonly` argument or `includeWatching` option now default to `true` for watch-only + wallets. Affected RPCs are: `getbalance`, `listreceivedbyaddress`, `listreceivedbylabel`, `listtransactions`, + `listsinceblock`, `gettransaction`, `walletcreatefundedpsbt`, and `fundrawtransaction`. (#16383) + +- `listunspent` now returns a "reused" bool for each output if the + wallet flag "avoid_reuse" is enabled. (#13756) + +- `getblockstats` now uses BlockUndo data instead of the transaction + index, making it much faster, no longer dependent on the `-txindex` + configuration option, and functional for all non-pruned blocks. + (#14802) + +- `utxoupdatepsbt` now accepts a `descriptors` parameter that will fill + out input and output scripts and keys when known. P2SH-witness inputs + will be filled in from the UTXO set when a descriptor is provided that + shows they're spending segwit outputs. See the RPC help text for full + details. (#15427) + +- `sendrawtransaction` and `testmempoolaccept` no longer accept a + `allowhighfees` parameter to fail mempool acceptance if the + transaction fee exceeds the value of the configuration option + `-maxtxfee`. Now there is a hardcoded default maximum feerate that + can be changed when calling either RPC using a `maxfeerate` parameter. + (#15620) + +- `getmempoolancestors`, `getmempooldescendants`, `getmempoolentry`, and + `getrawmempool` no longer return a `size` field unless the + configuration option `-deprecatedrpc=size` is used. Instead a new + `vsize` field is returned with the transaction's virtual size + (consistent with other RPCs such as `getrawtransaction`). (#15637) + +- `getwalletinfo` now includes a `scanning` field that is either `false` + (no scanning) or an object with information about the duration and + progress of the wallet's scanning historical blocks for transactions + affecting its balances. (#15730) + +- `gettransaction` now accepts a third (boolean) argument `verbose`. If + set to `true`, a new `decoded` field will be added to the response containing + the decoded transaction. This field is equivalent to RPC `decoderawtransaction`, + or RPC `getrawtransaction` when `verbose` is passed. (#16185, #16866, #16873) + +- `createwallet` accepts a new `passphrase` parameter. If set, this + will create the new wallet encrypted with the given passphrase. If + unset (the default) or set to an empty string, no encryption will be + used. (#16394) + +- `getchaintxstats` RPC now returns the additional key of + `window_final_block_height`. (#16695) + +- `getmempoolentry` now provides a `weight` field containing the + transaction weight as defined in BIP141. (#16647) + +- The `getnetworkinfo` and `getpeerinfo` commands now contain a new field with decoded network service flags. (#16786) + +- `getdescriptorinfo` now returns an additional `checksum` field + containing the checksum for the unmodified descriptor provided by the + user (that is, before the descriptor is normalized for the + `descriptor` field). (#15986) + +- `joinpsbts` now shuffles the order of the inputs and outputs of the resulting + joined PSBT. Previously, inputs and outputs were added in the order PSBTs were + provided. This made it easy to correlate inputs to outputs, representing a + privacy leak. (#16512) + +- `walletcreatefundedpsbt` now signals BIP125 Replace-by-Fee if the + `-walletrbf` configuration option is set to true. (#15911) + +GUI changes +----------- + +- The GUI wallet now provides bech32 addresses by default. The user may change the address type + during invoice generation using a GUI toggle, or the default address + type may be changed with the `-addresstype` configuration option. + (#15711, #16497) + +- In 0.18.0, a `./configure` flag was introduced to allow disabling BIP70 support in the GUI (support was enabled by default). In 0.19.0, this flag is now __disabled__ by default. If you want to compile Bitcoin Core with BIP70 support in the GUI, you can pass `--enable-bip70` to `./configure`. (#15584) + +Deprecated or removed configuration options +------------------------------------------- + +- `-mempoolreplacement` is removed, although default node behavior + remains the same. This option previously allowed the user to prevent + the node from accepting or relaying BIP125 transaction replacements. + This is different from the remaining configuration option + `-walletrbf`. (#16171) + +Deprecated or removed RPCs +-------------------------- + +- `bumpfee` no longer accepts a `totalFee` option unless the + configuration parameter `deprecatedrpc=totalFee` is specified. This + parameter will be fully removed in a subsequent release. (#15996) + +- `bumpfee` has a new `fee_rate` option as a replacement for the deprecated `totalFee`. (#16727) + +- `generate` is now removed after being deprecated in Bitcoin Core 0.18. + Use the `generatetoaddress` RPC instead. (#15492) + +P2P changes +----------- + +- BIP 61 reject messages were deprecated in v0.18. They are now disabled + by default, but can be enabled by setting the `-enablebip61` command + line option. BIP 61 reject messages will be removed entirely in a + future version of Bitcoin Core. (#14054) + +- To eliminate well-known denial-of-service vectors in Bitcoin Core, + especially for nodes with spinning disks, the default value for the + `-peerbloomfilters` configuration option has been changed to false. + This prevents Bitcoin Core from sending the BIP111 NODE_BLOOM service + flag, accepting BIP37 bloom filters, or serving merkle blocks or + transactions matching a bloom filter. Users who still want to provide + bloom filter support may either set the configuration option to true + to re-enable both BIP111 and BIP37 support or enable just BIP37 + support for specific peers using the updated `-whitelist` and + `-whitebind` configuration options described elsewhere in these + release notes. For the near future, lightweight clients using public + BIP111/BIP37 nodes should still be able to connect to older versions + of Bitcoin Core and nodes that have manually enabled BIP37 support, + but developers of such software should consider migrating to either + using specific BIP37 nodes or an alternative transaction filtering + system. (#16152) + +- By default, Bitcoin Core will now make two additional outbound connections that are exclusively used for block-relay. No transactions or addr messages will be processed on these connections. These connections are designed to add little additional memory or bandwidth resource requirements but should make some partitioning attacks more difficult to carry out. (#15759) + +Miscellaneous CLI Changes +------------------------- + +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to + `chain` and now returns the current network name as defined in BIP70 + (main, test, regtest). (#15566) + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` no longer returns a `bip9_softforks` object. + Instead, information has been moved into the `softforks` object and + an additional `type` field describes how Bitcoin Core determines + whether that soft fork is active (e.g. BIP9 or BIP90). See the RPC + help for details. (#16060) + +- `getblocktemplate` no longer returns a `rules` array containing `CSV` + and `segwit` (the BIP9 deployments that are currently in active + state). (#16060) + +- `getrpcinfo` now returns a `logpath` field with the path to + `debug.log`. (#15483) + +Tests +----- + +- The regression test chain enabled by the `-regtest` command line flag + now requires transactions to not violate standard policy by default. + This is the same default used for mainnet and makes it easier to test + mainnet behavior on regtest. Note that the testnet still allows + non-standard txs by default and that the policy can be locally + adjusted with the `-acceptnonstdtxn` command line flag for both test + chains. (#15891) + +Configuration +------------ + +- A setting specified in the default section but not also specified in a + network-specific section (e.g. testnet) will now produce an error + preventing startup instead of just a warning unless the network is + mainnet. This prevents settings intended for mainnet from being + applied to testnet or regtest. (#15629) + +- On platforms supporting `thread_local`, log lines can be prefixed with + the name of the thread that caused the log. To enable this behavior, + use `-logthreadnames=1`. (#15849) + +Network +------- + +- When fetching a transaction announced by multiple peers, previous versions of + Bitcoin Core would sequentially attempt to download the transaction from each + announcing peer until the transaction is received, in the order that those + peers' announcements were received. In this release, the download logic has + changed to randomize the fetch order across peers and to prefer sending + download requests to outbound peers over inbound peers. This fixes an issue + where inbound peers could prevent a node from getting a transaction. + (#14897, #15834) + +- If a Tor hidden service is being used, Bitcoin Core will be bound to + the standard port 8333 even if a different port is configured for + clearnet connections. This prevents leaking node identity through use + of identical non-default port numbers. (#15651) + +Mempool and transaction relay +----------------------------- + +- Allows one extra single-ancestor transaction per package. Previously, + if a transaction in the mempool had 25 descendants, or it and all of + its descendants were over 101,000 vbytes, any newly-received + transaction that was also a descendant would be ignored. Now, one + extra descendant will be allowed provided it is an immediate + descendant (child) and the child's size is 10,000 vbytes or less. + This makes it possible for two-party contract protocols such as + Lightning Network to give each participant an output they can spend + immediately for Child-Pays-For-Parent (CPFP) fee bumping without + allowing one malicious participant to fill the entire package and thus + prevent the other participant from spending their output. (#15681) + +- Transactions with outputs paying v1 to v16 witness versions (future + segwit versions) are now accepted into the mempool, relayed, and + mined. Attempting to spend those outputs remains forbidden by policy + ("non-standard"). When this change has been widely deployed, wallets + and services can accept any valid bech32 Bitcoin address without + concern that transactions paying future segwit versions will become + stuck in an unconfirmed state. (#15846) + +- Legacy transactions (transactions with no segwit inputs) must now be + sent using the legacy encoding format, enforcing the rule specified in + BIP144. (#14039) + +Wallet +------ + +- When in pruned mode, a rescan that was triggered by an `importwallet`, + `importpubkey`, `importaddress`, or `importprivkey` RPC will only fail + when blocks have been pruned. Previously it would fail when `-prune` + has been set. This change allows setting `-prune` to a high value + (e.g. the disk size) without the calls to any of the import RPCs + failing until the first block is pruned. (#15870) + +- When creating a transaction with a fee above `-maxtxfee` (default 0.1 + BTC), the RPC commands `walletcreatefundedpsbt` and + `fundrawtransaction` will now fail instead of rounding down the fee. + Be aware that the `feeRate` argument is specified in BTC per 1,000 + vbytes, not satoshi per vbyte. (#16257) + +- A new wallet flag `avoid_reuse` has been added (default off). When + enabled, a wallet will distinguish between used and unused addresses, + and default to not use the former in coin selection. When setting + this flag on an existing wallet, rescanning the blockchain is required + to correctly mark previously used destinations. Together with "avoid + partial spends" (added in Bitcoin Core v0.17.0), this can eliminate a + serious privacy issue where a malicious user can track spends by + sending small payments to a previously-paid address that would then + be included with unrelated inputs in future payments. (#13756) + +Build system changes +-------------------- + +- Python >=3.5 is now required by all aspects of the project. This + includes the build systems, test framework and linters. The previously + supported minimum (3.4), was EOL in March 2019. (#14954) + +- The minimum supported miniUPnPc API version is set to 10. This keeps + compatibility with Ubuntu 16.04 LTS and Debian 8 `libminiupnpc-dev` + packages. Please note, on Debian this package is still vulnerable to + [CVE-2017-8798](https://security-tracker.debian.org/tracker/CVE-2017-8798) + (in jessie only) and + [CVE-2017-1000494](https://security-tracker.debian.org/tracker/CVE-2017-1000494) + (both in jessie and in stretch). (#15993) + +0.19.0 change log +================= + +### Consensus +- #16128 Delete error-prone CScript constructor only used with FindAndDelete (instagibbs) +- #16060 Bury bip9 deployments (jnewbery) + +### Policy +- #15557 Enhance `bumpfee` to include inputs when targeting a feerate (instagibbs) +- #15846 Make sending to future native witness outputs standard (sipa) + +### Block and transaction handling +- #15632 Remove ResendWalletTransactions from the Validation Interface (jnewbery) +- #14121 Index for BIP 157 block filters (jimpo) +- #15141 Rewrite DoS interface between validation and net_processing (sdaftuar) +- #15880 utils and libraries: Replace deprecated Boost Filesystem functions (hebasto) +- #15971 validation: Add compile-time checking for negative locking requirement in LimitValidationInterfaceQueue (practicalswift) +- #15999 init: Remove dead code in LoadChainTip (MarcoFalke) +- #16015 validation: Hold cs_main when reading chainActive in RewindBlockIndex (practicalswift) +- #16056 remove unused magic number from consistency check (instagibbs) +- #16171 Remove -mempoolreplacement to prevent needless block prop slowness (TheBlueMatt) +- #15894 Remove duplicated "Error: " prefix in logs (hebasto) +- #14193 validation: Add missing mempool locks (MarcoFalke) +- #15681 Allow one extra single-ancestor transaction per package (TheBlueMatt) +- #15305 [validation] Crash if disconnecting a block fails (sdaftuar) +- #16471 log correct messages when CPFP fails (jnewbery) +- #16433 txmempool: Remove unused default value MemPoolRemovalReason::UNKNOWN (MarcoFalke) +- #13868 Remove unused fScriptChecks parameter from CheckInputs (Empact) +- #16421 Conservatively accept RBF bumps bumping one tx at the package limits (TheBlueMatt) +- #16854 Prevent UpdateTip log message from being broken up (stevenroose) +- #16956 validation: Make GetWitnessCommitmentIndex public (MarcoFalke) +- #16713 Ignore old versionbit activations to avoid 'unknown softforks' warning (jnewbery) +- #17002 chainparams: Bump assumed chain params (MarcoFalke) +- #16849 Fix block index inconsistency in InvalidateBlock() (sdaftuar) + +### P2P protocol and network code +- #15597 Generate log entry when blocks messages are received unexpectedly (pstratem) +- #15654 Remove unused unsanitized user agent string CNode::strSubVer (MarcoFalke) +- #15689 netaddress: Update CNetAddr for ORCHIDv2 (dongcarl) +- #15834 Fix transaction relay bugs introduced in #14897 and expire transactions from peer in-flight map (sdaftuar) +- #15651 torcontrol: Use the default/standard network port for Tor hidden services, even if the internal port is set differently (luke-jr) +- #16188 Document what happens to getdata of unknown type (MarcoFalke) +- #15649 Add ChaCha20Poly1305@Bitcoin AEAD (jonasschnelli) +- #16152 Disable bloom filtering by default (TheBlueMatt) +- #15993 Drop support of the insecure miniUPnPc versions (hebasto) +- #16197 Use mockable time for tx download (MarcoFalke) +- #16248 Make whitebind/whitelist permissions more flexible (NicolasDorier) +- #16618 [Fix] Allow connection of a noban banned peer (NicolasDorier) +- #16631 Restore default whitelistrelay to true (NicolasDorier) +- #15759 Add 2 outbound block-relay-only connections (sdaftuar) +- #15558 Don't query all DNS seeds at once (sipa) +- #16999 0.19 seeds update (laanwj) + +### Wallet +- #15288 Remove wallet -> node global function calls (ryanofsky) +- #15491 Improve log output for errors during load (gwillen) +- #13541 wallet/rpc: sendrawtransaction maxfeerate (kallewoof) +- #15680 Remove resendwallettransactions RPC method (jnewbery) +- #15508 Refactor analyzepsbt for use outside RPC code (gwillen) +- #15747 Remove plethora of Get*Balance (MarcoFalke) +- #15728 Refactor relay transactions (jnewbery) +- #15639 bitcoin-wallet tool: Drop libbitcoin_server.a dependency (ryanofsky) +- #15853 Remove unused import checkpoints.h (MarcoFalke) +- #15780 add cachable amounts for caching credit/debit values (kallewoof) +- #15778 Move maxtxfee from node to wallet (jnewbery) +- #15901 log on rescan completion (andrewtoth) +- #15917 Avoid logging no_such_file_or_directory error (promag) +- #15452 Replace CScriptID and CKeyID in CTxDestination with dedicated types (instagibbs) +- #15870 Only fail rescan when blocks have actually been pruned (MarcoFalke) +- #15006 Add option to create an encrypted wallet (achow101) +- #16001 Give WalletModel::UnlockContext move semantics (sipa) +- #15741 Batch write imported stuff in importmulti (achow101) +- #16144 do not encrypt wallets with disabled private keys (mrwhythat) +- #15024 Allow specific private keys to be derived from descriptor (meshcollider) +- #13756 "avoid_reuse" wallet flag for improved privacy (kallewoof) +- #16226 Move ismine to the wallet module (achow101) +- #16239 wallet/rpc: follow-up clean-up/fixes to avoid_reuse (kallewoof) +- #16286 refactoring: wallet: Fix GCC 7.4.0 warning (hebasto) +- #16257 abort when attempting to fund a transaction above -maxtxfee (Sjors) +- #16237 Have the wallet give out destinations instead of keys (achow101) +- #16322 Fix -maxtxfee check by moving it to CWallet::CreateTransaction (promag) +- #16361 Remove redundant pre-TopUpKeypool check (instagibbs) +- #16244 Move wallet creation out of the createwallet rpc into its own function (achow101) +- #16227 Refactor CWallet's inheritance chain (achow101) +- #16208 Consume ReserveDestination on successful CreateTransaction (instagibbs) +- #16301 Use CWallet::Import* functions in all import* RPCs (achow101) +- #16402 Remove wallet settings from chainparams (MarcoFalke) +- #16415 Get rid of PendingWalletTx class (ryanofsky) +- #15588 Log the actual wallet file version and no longer publicly expose the "version" record (achow101) +- #16399 Improve wallet creation (fjahr) +- #16475 Enumerate walletdb keys (MarcoFalke) +- #15709 Do not add "setting" key as unknown (Bushstar) +- #16451 Remove CMerkleTx (jnewbery) +- #15906 Move min_depth and max_depth to coin control (amitiuttarwar) +- #16502 Drop unused OldKey (promag) +- #16394 Allow createwallet to take empty passwords to make unencrypted wallets (achow101) +- #15911 Use wallet RBF default for walletcreatefundedpsbt (Sjors) +- #16503 Remove p2pEnabled from Chain interface (ariard) +- #16557 restore coinbase and confirmed/conflicted checks in SubmitMemoryPoolAndRelay() (jnewbery) +- #14934 Descriptor expansion cache clarifications (Sjors) +- #16383 rpcwallet: default include_watchonly to true for watchonly wallets (jb55) +- #16542 Return more specific errors about invalid descriptors (achow101) +- #16572 Fix Char as Bool in Wallet (JeremyRubin) +- #16753 extract PubKey from P2PK script with Solver (theStack) +- #16716 Use wallet name instead of pointer on unload/release (promag) +- #16185 gettransaction: add an argument to decode the transaction (darosior) +- #16745 Translate all initErrors in CreateWalletFromFile (MarcoFalke) +- #16792 Assert that the HRP is lowercase in Bech32::Encode (meshcollider) +- #16624 encapsulate transactions state (ariard) +- #16830 Cleanup walletinitinterface.h (hebasto) +- #16796 Fix segfault in CreateWalletFromFile (MarcoFalke) +- #16866 Rename 'decode' argument in gettransaction method to 'verbose' (jnewbery) +- #16727 Explicit feerate for bumpfee (instagibbs) +- #16609 descriptor: fix missed m_script_arg arg renaming in #14934 (fanquake) + +### RPC and other APIs +- #15492 remove deprecated generate method (Sjors) +- #15566 cli: Replace testnet with chain and return network name as per bip70 (fanquake) +- #15564 cli: Remove duplicate wallet fields from -getinfo (fanquake) +- #15642 Remove deprecated rpc warnings (jnewbery) +- #15637 Rename size to vsize in mempool related calls (fanquake) +- #15620 Uncouple non-wallet rpcs from maxTxFee global (MarcoFalke) +- #15616 Clarify decodescript RPCResult doc (MarcoFalke) +- #15669 Fix help text for signtransactionwithXXX (torkelrogstad) +- #15596 Ignore sendmany::minconf as dummy value (MarcoFalke) +- #15755 remove unused var in rawtransaction.cpp (Bushstar) +- #15746 RPCHelpMan: Always name dictionary keys (MarcoFalke) +- #15748 remove dead mining code (jnewbery) +- #15751 Speed up deriveaddresses for large ranges (sipa) +- #15770 Validate maxfeerate with AmountFromValue (promag) +- #15474 rest/rpc: Make mempoolinfo atomic (promag) +- #15463 Speedup getaddressesbylabel (promag) +- #15784 Remove dependency on interfaces::Chain in SignTransaction (ariard) +- #15323 Expose g_is_mempool_loaded via getmempoolinfo (Empact) +- #15932 Serialize in getblock without cs_main (MarcoFalke) +- #15930 Add balances RPC (MarcoFalke) +- #15730 Show scanning details in getwalletinfo (promag) +- #14802 faster getblockstats using BlockUndo data (FelixWeis) +- #14984 Speedup getrawmempool when verbose=true (promag) +- #16071 Hint for importmulti in help output of importpubkey and importaddress (kristapsk) +- #16063 Mention getwalletinfo where a rescan is triggered (promag) +- #16024 deriveaddresses: Correction of descriptor checksum in RPC example (ccapo) +- #16217 getrawtransaction: inform about blockhash argument when lookup fails (darosior) +- #15427 Add support for descriptors to utxoupdatepsbt (sipa) +- #16262 Allow shutdown while in generateblocks (pstratem) +- #15483 Adding a 'logpath' entry to getrpcinfo (darosior) +- #16325 Clarify that block count means height excl genesis (MarcoFalke) +- #16326 add new utxoupdatepsbt arguments to the CRPCCommand and CPRCCvertParam tables (jnewbery) +- #16332 Add logpath description for getrpcinfo (instagibbs) +- #16240 JSONRPCRequest-aware RPCHelpMan (kallewoof) +- #15996 Deprecate totalfee argument in `bumpfee` (instagibbs) +- #16467 sendrawtransaction help privacy note (jonatack) +- #16596 Fix getblocktemplate CLI example (emilengler) +- #15986 Add checksum to getdescriptorinfo (sipa) +- #16647 add weight to getmempoolentry output (fanquake) +- #16695 Add window final block height to getchaintxstats (leto) +- #16798 Refactor rawtransaction_util's SignTransaction to separate prevtx parsing (achow101) +- #16285 Improve scantxoutset response and help message (promag) +- #16725 Don't show addresses or P2PK in decoderawtransaction (NicolasDorier) +- #16787 Human readable network services (darosior) +- #16251 Improve signrawtransaction error reporting (ajtowns) +- #16873 fix regression in gettransaction (jonatack) +- #16512 Shuffle inputs and outputs after joining psbts (achow101) +- #16521 Use the default maxfeerate value as BTC/kB (Remagpie) +- #16817 Fix casing in getblockchaininfo to be inline with other fields (dangershony) +- #17131 fix -rpcclienttimeout 0 option (fjahr) +- #17249 Add missing deque include to fix build (jbeich) +- #17368 cli: fix -getinfo output when compiled with no wallet (fanquake) + +### GUI +- #15464 Drop unused return values in WalletFrame (promag) +- #15614 Defer removeAndDeleteWallet when no modal widget is active (promag) +- #15711 Generate bech32 addresses by default (MarcoFalke) +- #15829 update request payment button text and tab description (fanquake) +- #15874 Resolve the qt/guiutil <-> qt/optionsmodel CD (251Labs) +- #15371 Uppercase bech32 addresses in qr codes (benthecarman) +- #15928 Move QRImageWidget to its own file-pair (luke-jr) +- #16113 move coin control "OK" to the right hand side of the dialog (fanquake) +- #16090 Add vertical spacer to peer detail widget (JosuGZ) +- #15886 qt, wallet: Revamp SendConfirmationDialog (hebasto) +- #16263 Use qInfo() if no error occurs (hebasto) +- #16153 Add antialiasing to traffic graph widget (JosuGZ) +- #16350 Remove unused guard (hebasto) +- #16106 Sort wallets in open wallet menu (promag) +- #16291 Stop translating PACKAGE_NAME (MarcoFalke) +- #16380 Remove unused bits from the service flags enum (MarcoFalke) +- #16379 Fix autostart filenames on Linux for testnet/regtest (hebasto) +- #16366 init: Use InitError for all errors in bitcoind/qt (MarcoFalke) +- #16436 Do not create payment server if -disablewallet option provided (hebasto) +- #16514 Remove unused RPCConsole::tabFocus (promag) +- #16497 Generate bech32 addresses by default (take 2, fixup) (MarcoFalke) +- #16349 Remove redundant WalletController::addWallet slot (hebasto) +- #16578 Do not pass in command line arguments to QApplication (achow101) +- #16612 Remove menu icons (laanwj) +- #16677 remove unused PlatformStyle::TextColorIcon (fanquake) +- #16694 Ensure transaction send error is always visible (fanquake) +- #14879 Add warning messages to the debug window (hebasto) +- #16708 Replace obsolete functions of QSslSocket (hebasto) +- #16701 Replace functions deprecated in Qt 5.13 (hebasto) +- #16706 Replace deprecated QSignalMapper by lambda expressions (hebasto) +- #16707 Remove obsolete QModelIndex::child() (hebasto) +- #16758 Replace QFontMetrics::width() with TextWidth() (hebasto) +- #16760 Change uninstall icon on Windows (GChuf) +- #16720 Replace objc_msgSend() function calls with the native Objective-C syntax (hebasto) +- #16788 Update transifex slug for 0.19 (laanwj) +- #15450 Create wallet menu option (achow101) +- #16735 Remove unused menu items for Windows and Linux (GChuf) +- #16826 Do additional character escaping for wallet names and address labels (achow101) +- #15529 Add Qt programs to msvc build (updated, no code changes) (sipsorcery) +- #16714 add prune to intro screen with smart default (Sjors) +- #16858 advise users not to switch wallets when opening a BIP70 URI (jameshilliard) +- #16822 Create wallet menu option follow-ups (jonatack) +- #16882 Re-generate translations before 0.19.0 (MarcoFalke) +- #16928 Rename address checkbox back to bech32 (MarcoFalke) +- #16837 Fix {C{,XX},LD}FLAGS pickup (dongcarl) +- #16971 Change default size of intro frame (emilengler) +- #16988 Periodic translations update (laanwj) +- #16852 When BIP70 is disabled, get PaymentRequest merchant using string search (achow101) +- #16952 make sure to update the UI when deleting a transaction (jonasschnelli) +- #17031 Prevent processing duplicate payment requests (promag) +- #17135 Make polling in ClientModel asynchronous (promag) +- #17120 Fix start timer from non QThread (promag) +- #17257 disable font antialiasing for QR image address (fanquake) + +### Build system +- #14954 Require python 3.5 (MarcoFalke) +- #15580 native_protobuf: avoid system zlib (dongcarl) +- #15601 Switch to python3 (take 3) (MarcoFalke) +- #15581 Make less assumptions about build env (dongcarl) +- #14853 latest RapidCheck (fanquake) +- #15446 Improve depends debuggability (dongcarl) +- #13788 Fix --disable-asm for newer assembly checks/code (luke-jr) +- #12051 add missing debian contrib file to tarball (puchu) +- #15919 Remove unused OpenSSL includes to make it more clear where OpenSSL is used (practicalswift) +- #15978 .gitignore: Don't ignore depends patches (dongcarl) +- #15939 gitian: Remove windows 32 bit build (MarcoFalke) +- #15239 scripts and tools: Move non-linux build source tarballs to "bitcoin-binaries/version" directory (hebasto) +- #14047 Add HKDF_HMAC256_L32 and method to negate a private key (jonasschnelli) +- #16051 add patch to common dependencies (fanquake) +- #16049 switch to secure download of all dependencies (Kemu) +- #16059 configure: Fix thread_local detection (dongcarl) +- #16089 add ability to skip building zeromq (fanquake) +- #15844 Purge libtool archives (dongcarl) +- #15461 update to Boost 1.70 (Sjors) +- #16141 remove GZIP export from gitian descriptors (fanquake) +- #16235 Cleaned up and consolidated msbuild files (no code changes) (sipsorcery) +- #16246 MSVC: Fix error in debug mode (Fix #16245) (NicolasDorier) +- #16183 xtrans: Configure flags cleanup (dongcarl) +- #16258 [MSVC]: Create the config.ini as part of bitcoind build (NicolasDorier) +- #16271 remove -Wall from rapidcheck build flags (fanquake) +- #16309 [MSVC] allow user level project customization (NicolasDorier) +- #16308 [MSVC] Copy build output to src/ automatically after build (NicolasDorier) +- #15457 Check std::system for -[alert|block|wallet]notify (Sjors) +- #16344 use #if HAVE_SYSTEM instead of defined(HAVE_SYSTEM) (Sjors) +- #16352 prune dbus from depends (fanquake) +- #16270 expat 2.2.7 (fanquake) +- #16408 Prune X packages (dongcarl) +- #16386 disable unused Qt features (fanquake) +- #16424 Treat -Wswitch as error when --enable-werror (MarcoFalke) +- #16441 remove qt libjpeg check from bitcoin_qt.m4 (fanquake) +- #16434 Specify AM_CPPFLAGS for ZMQ (domob1812) +- #16534 add Qt Creator Makefile.am.user to .gitignore (Bushstar) +- #16573 disable building libsecp256k1 benchmarks (fanquake) +- #16533 disable libxcb extensions (fanquake) +- #16589 Remove unused src/obj-test folder (MarcoFalke) +- #16435 autoconf: Sane `--enable-debug` defaults (dongcarl) +- #16622 echo property tests status during build (jonatack) +- #16611 Remove src/obj directory from repository (laanwj) +- #16371 ignore macOS make deploy artefacts & add them to clean-local (fanquake) +- #16654 build: update RapidCheck Makefile (jonatack) +- #16370 cleanup package configure flags (fanquake) +- #16746 msbuild: Ignore linker warning (sipsorcery) +- #16750 msbuild: adds bench_bitcoin to auto generated project files (sipsorcery) +- #16810 guix: Remove ssp spec file hack (dongcarl) +- #16477 skip deploying plugins we dont use in macdeployqtplus (fanquake) +- #16413 Bump QT to LTS release 5.9.8 (THETCR) +- #15584 disable BIP70 support by default (fanquake) +- #16871 make building protobuf optional in depends (fanquake) +- #16879 remove redundant sed patching (fanquake) +- #16809 zlib: Move toolchain options to configure (dongcarl) +- #15146 Solve SmartOS FD_ZERO build issue (Empact) +- #16870 update boost macros to latest upstream for improved error reporting (fanquake) +- #16982 Factor out qt translations from build system (laanwj) +- #16926 Add OpenSSL termios fix for musl libc (nmarley) +- #16927 Refresh ZeroMQ 4.3.1 patch (nmarley) +- #17005 Qt version appears only if GUI is being built (ch4ot1c) +- #16468 Exclude depends/Makefile in .gitignore (promag) + +### Tests and QA +- #15296 Add script checking for deterministic line coverage in unit tests (practicalswift) +- #15338 ci: Build and run tests once on freebsd (MarcoFalke) +- #15479 Add .style.yapf (MarcoFalke) +- #15534 lint-format-strings: open files sequentially (fix for OS X) (gwillen) +- #15504 fuzz: Link BasicTestingSetup (shared with unit tests) (MarcoFalke) +- #15473 bench: Benchmark mempooltojson (MarcoFalke) +- #15466 Print remaining jobs in test_runner.py (stevenroose) +- #15631 mininode: Clearer error message on invalid magic bytes (MarcoFalke) +- #15255 Remove travis_wait from lint script (gkrizek) +- #15686 make pruning test faster (jnewbery) +- #15533 .style.yapf: Set column_limit=160 (MarcoFalke) +- #15660 Overhaul p2p_compactblocks.py (sdaftuar) +- #15495 Add regtests for HTTP status codes (domob1812) +- #15772 Properly log named args in authproxy (MarcoFalke) +- #15771 Prevent concurrency issues reading .cookie file (promag) +- #15693 travis: Switch to ubuntu keyserver to avoid timeouts (MarcoFalke) +- #15629 init: Throw error when network specific config is ignored (MarcoFalke) +- #15773 Add BitcoinTestFramework::sync_* methods (MarcoFalke) +- #15797 travis: Bump second timeout to 33 minutes, add rationale (MarcoFalke) +- #15788 Unify testing setups for fuzz, bench, and unit tests (MarcoFalke) +- #15352 Reduce noise level in test_bitcoin output (practicalswift) +- #15779 Add wallet_balance benchmark (MarcoFalke) +- #15843 fix outdated include in blockfilter_index_tests (jamesob) +- #15866 Add missing syncwithvalidationinterfacequeue to wallet_import_rescan (MarcoFalke) +- #15697 Make swap_magic_bytes in p2p_invalid_messages atomic (MarcoFalke) +- #15895 Avoid re-reading config.ini unnecessarily (luke-jr) +- #15896 feature_filelock, interface_bitcoin_cli: Use PACKAGE_NAME in messages rather than hardcoding Bitcoin Core (luke-jr) +- #15897 QA/mininode: Send all headers upfront in send_blocks_and_test to avoid sending an unconnected one (luke-jr) +- #15696 test_runner: Move feature_pruning to base tests (MarcoFalke) +- #15869 Add settings merge test to prevent regresssions (ryanofsky) +- #15758 Add further tests to wallet_balance (MarcoFalke) +- #15841 combine_logs: append node stderr and stdout if it exists (MarcoFalke) +- #15949 test_runner: Move pruning back to extended (MarcoFalke) +- #15927 log thread names by default in functional tests (jnewbery) +- #15664 change default Python block serialization to witness (instagibbs) +- #15988 Add test for ArgsManager::GetChainName (ryanofsky) +- #15963 Make random seed logged and settable (jnewbery) +- #15943 Fail if RPC has been added without tests (MarcoFalke) +- #16036 travis: Run all lint scripts even if one fails (scravy) +- #13555 parameterize adjustment period in versionbits_computeblockversion (JBaczuk) +- #16079 wallet_balance.py: Prevent edge cases (stevenroose) +- #16078 replace tx hash with txid in rawtransaction test (LongShao007) +- #16042 Bump MAX_NODES to 12 (MarcoFalke) +- #16124 Limit Python linting to files in the repo (practicalswift) +- #16143 Mark unit test blockfilter_index_initial_sync as non-deterministic (practicalswift) +- #16214 travis: Fix caching issues (MarcoFalke) +- #15982 Make msg_block a witness block (MarcoFalke) +- #16225 Make coins_tests/updatecoins_simulation_test deterministic (practicalswift) +- #16236 fuzz: Log output even if fuzzer failed (MarcoFalke) +- #15520 cirrus: Run extended test feature_pruning (MarcoFalke) +- #16234 Add test for unknown args (MarcoFalke) +- #16207 stop generating lcov coverage when functional tests fail (asood123) +- #16252 Log to debug.log in all unit tests (MarcoFalke) +- #16289 Add missing ECC_Stop() in GUI rpcnestedtests.cpp (jonasschnelli) +- #16278 Remove unused includes (practicalswift) +- #16302 Add missing syncwithvalidationinterfacequeue to wallet_balance test (MarcoFalke) +- #15538 wallet_bumpfee.py: Make sure coin selection produces change (instagibbs) +- #16294 Create at most one testing setup (MarcoFalke) +- #16299 bench: Move generated data to a dedicated translation unit (promag) +- #16329 Add tests for getblockchaininfo.softforks (MarcoFalke) +- #15687 tool wallet test coverage for unexpected writes to wallet (jonatack) +- #16267 bench: Benchmark blocktojson (fanatid) +- #14505 Add linter to make sure single parameter constructors are marked explicit (practicalswift) +- #16338 Disable other targets when enable-fuzz is set (qmma70) +- #16334 rpc_users: Also test rpcauth.py with password (dongcarl) +- #15282 Replace hard-coded hex tx with class in test framework (stevenroose) +- #16390 Add --filter option to test_runner.py (promag) +- #15891 Require standard txs in regtest by default (MarcoFalke) +- #16374 Enable passing wildcard test names to test runner from root (jonatack) +- #16420 Fix race condition in wallet_encryption test (jonasschnelli) +- #16422 remove redundant setup in addrman_tests (zenosage) +- #16438 travis: Print memory and number of cpus (MarcoFalke) +- #16445 Skip flaky p2p_invalid_messages test on macOS (fjahr) +- #16459 Fix race condition in example_test.py (sdaftuar) +- #16464 Ensure we don't generate a too-big block in p2sh sigops test (sdaftuar) +- #16491 fix deprecated log.warn in feature_dbcrash test (jonatack) +- #15134 Switch one of the Travis jobs to an unsigned char environment (-funsigned-char) (practicalswift) +- #16505 Changes verbosity of msbuild from quiet to normal in the appveyor script (sipsorcery) +- #16293 Make test cases separate functions (MarcoFalke) +- #16470 Fail early on disconnect in mininode.wait_for_* (MarcoFalke) +- #16277 Suppress output in test_bitcoin for expected errors (gertjaap) +- #16493 Fix test failures (MarcoFalke) +- #16538 Add missing sync_blocks to feature_pruning (MarcoFalke) +- #16509 Adapt test framework for chains other than "regtest" (MarcoFalke) +- #16363 Add test for BIP30 duplicate tx (MarcoFalke) +- #16535 Explain why -whitelist is used in feature_fee_estimation (MarcoFalke) +- #16554 only include and use OpenSSL where it's actually needed (BIP70) (fanquake) +- #16598 Remove confusing hash256 function in util (elichai) +- #16595 travis: Use extended 90 minute timeout when available (MarcoFalke) +- #16563 Add unit test for AddTimeData (mzumsande) +- #16561 Use colors and dots in test_runner.py output only if standard output is a terminal (practicalswift) +- #16465 Test p2sh-witness and bech32 in wallet_import_rescan (MarcoFalke) +- #16582 Rework ci (Use travis only as fallback env) (MarcoFalke) +- #16633 travis: Fix test_runner.py timeouts (MarcoFalke) +- #16646 Run tests with UPnP disabled (fanquake) +- #16623 ci: Add environment files for all settings (MarcoFalke) +- #16656 fix rpc_setban.py race (jonasschnelli) +- #16570 Make descriptor tests deterministic (davereikher) +- #16404 Test ZMQ notification after chain reorg (promag) +- #16726 Avoid common Python default parameter gotcha when mutable dict/list:s are used as default parameter values (practicalswift) +- #16739 ci: Pass down $makejobs to test_runner.py, other improvements (MarcoFalke) +- #16767 Check for codespell in lint-spelling.sh (kristapsk) +- #16768 Make lint-includes.sh work from any directory (kristapsk) +- #15257 Scripts and tools: Bump flake8 to 3.7.8 (Empact) +- #16804 Remove unused try-block in assert_debug_log (MarcoFalke) +- #16850 `servicesnames` field in `getpeerinfo` and `getnetworkinfo` (darosior) +- #16551 Test that low difficulty chain fork is rejected (MarcoFalke) +- #16737 Establish only one connection between nodes in rpc_invalidateblock (MarcoFalke) +- #16845 Add notes on how to generate data/wallets/high_minversion (MarcoFalke) +- #16888 Bump timeouts in slow running tests (MarcoFalke) +- #16864 Add python bech32 impl round-trip test (instagibbs) +- #16865 add some unit tests for merkle.cpp (soroosh-sdi) +- #14696 Add explicit references to related CVE's in p2p_invalid_block test (lucash-dev) +- #16907 lint: Add DisabledOpcodeTemplates to whitelist (MarcoFalke) +- #16898 Remove connect_nodes_bi (MarcoFalke) +- #16917 Move common function assert_approx() into util.py (fridokus) +- #16921 Add information on how to add Vulture suppressions (practicalswift) +- #16920 Fix extra_args in wallet_import_rescan.py (MarcoFalke) +- #16918 Make PORT_MIN in test runner configurable (MarcoFalke) +- #16941 travis: Disable feature_block in tsan run due to oom (MarcoFalke) +- #16929 follow-up to rpc: default maxfeerate value as BTC/kB (jonatack) +- #16959 ci: Set $host before setting fallback values (MarcoFalke) +- #16961 Remove python dead code linter (laanwj) +- #16931 add unittests for CheckProofOfWork (soroosh-sdi) +- #16991 Fix service flag comparison check in rpc_net test (luke-jr) (laanwj) +- #16987 Correct docstring param name (jbampton) +- #17015 Explain QT_QPA_PLATFORM for gui tests (MarcoFalke) +- #17006 Enable UBSan for Travis fuzzing job (practicalswift) +- #17086 Fix fs_tests for unknown locales (carnhofdaki) +- #15903 appveyor: Write @PACKAGE_NAME@ to config (MarcoFalke) +- #16742 test: add executable flag for wallet_watchonly.py (theStack) +- #16740 qa: Relax so that the subscriber is ready before publishing zmq messages (#16740) + +### Miscellaneous +- #15335 Fix lack of warning of unrecognized section names (AkioNak) +- #15528 contrib: Bump gitian descriptors for 0.19 (MarcoFalke) +- #15609 scripts and tools: Set 'distro' explicitly (hebasto) +- #15519 Add Poly1305 implementation (jonasschnelli) +- #15643 contrib: Gh-merge: include acks in merge commit (MarcoFalke) +- #15838 scripts and tools: Fetch missing review comments in github-merge.py (nkostoulas) +- #15920 lint: Check that all wallet args are hidden (MarcoFalke) +- #15849 Thread names in logs and deadlock debug tools (jamesob) +- #15650 Handle the result of posix_fallocate system call (lucayepa) +- #15766 scripts and tools: Upgrade gitian image before signing (hebasto) +- #15512 Add ChaCha20 encryption option (XOR) (jonasschnelli) +- #15968 Fix portability issue with pthreads (grim-trigger) +- #15970 Utils and libraries: fix static_assert for macro HAVE_THREAD_LOCAL (orientye) +- #15863 scripts and tools: Ensure repos are up-to-date in gitian-build.py (hebasto) +- #15224 Add RNG strengthening (10ms once every minute) (sipa) +- #15840 Contrib scripts: Filter IPv6 by ASN (abitfan) +- #13998 Scripts and tools: gitian-build.py improvements and corrections (hebasto) +- #15236 scripts and tools: Make --setup command independent (hebasto) +- #16114 contrib: Add curl as a required program in gitian-build.py (fanquake) +- #16046 util: Add type safe gettime (MarcoFalke) +- #15703 Update secp256k1 subtree to latest upstream (sipa) +- #16086 contrib: Use newer config.guess & config.sub in install_db4.sh (fanquake) +- #16130 Don't GPG sign intermediate commits with github-merge tool (stevenroose) +- #16162 scripts: Add key for michael ford (fanquake) to trusted keys list (fanquake) +- #16201 devtools: Always use unabbreviated commit IDs in github-merge.py (laanwj) +- #16112 util: Log early messages (MarcoFalke) +- #16223 devtools: Fetch and display ACKs at sign-off time in github-merge (laanwj) +- #16300 util: Explain why the path is cached (MarcoFalke) +- #16314 scripts and tools: Update copyright_header.py script (hebasto) +- #16158 Fix logic of memory_cleanse() on MSVC and clean up docs (real-or-random) +- #14734 fix an undefined behavior in uint::SetHex (kazcw) +- #16327 scripts and tools: Update ShellCheck linter (hebasto) +- #15277 contrib: Enable building in guix containers (dongcarl) +- #16362 Add bilingual_str type (hebasto) +- #16481 logs: add missing space (harding) +- #16581 sipsorcery gitian key (sipsorcery) +- #16566 util: Refactor upper/lowercase functions (kallewoof) +- #16620 util: Move resolveerrmsg to util/error (MarcoFalke) +- #16625 scripts: Remove github-merge.py (fanquake) +- #15864 Fix datadir handling (hebasto) +- #16670 util: Add join helper to join a list of strings (MarcoFalke) +- #16665 scripts: Move update-translations.py to maintainer-tools repo (fanquake) +- #16730 Support serialization of `std::vector<bool>` (sipa) +- #16556 Fix systemd service file configuration directory setup (setpill) +- #15615 Add log output during initial header sync (jonasschnelli) +- #16774 Avoid unnecessary "Synchronizing blockheaders" log messages (jonasschnelli) +- #16489 log: harmonize bitcoind logging (jonatack) +- #16577 util: Cbufferedfile fixes and unit test (LarryRuane) +- #16984 util: Make thread names shorter (hebasto) +- #17038 Don't rename main thread at process level (laanwj) +- #17184 util: Filter out macos process serial number (hebasto) +- #17095 util: Filter control characters out of log messages (laanwj) +- #17085 init: Change fallback locale to C.UTF-8 (laanwj) +- #16957 9% less memory: make SaltedOutpointHasher noexcept (martinus) +- #17449 fix uninitialized variable nMinerConfirmationWindow (bitcoinVBR) + +### Documentation +- #15514 Update Transifex links (fanquake) +- #15513 add "sections" info to example bitcoin.conf (fanquake) +- #15530 Move wallet lock annotations to header (MarcoFalke) +- #15562 remove duplicate clone step in build-windows.md (fanquake) +- #15565 remove release note fragments (fanquake) +- #15444 Additional productivity tips (Sjors) +- #15577 Enable TLS in link to chris.beams.io (JeremyRand) +- #15604 release note for disabling reject messages by default (jnewbery) +- #15611 Add Gitian key for droark (droark) +- #15626 Update ACK description in CONTRIBUTING.md (jonatack) +- #15603 Add more tips to productivity.md (gwillen) +- #15683 Comment for seemingly duplicate LIBBITCOIN_SERVER (Bushstar) +- #15685 rpc-mining: Clarify error messages (MarcoFalke) +- #15760 Clarify sendrawtransaction::maxfeerate==0 help (MarcoFalke) +- #15659 fix findFork comment (r8921039) +- #15718 Improve netaddress comments (dongcarl) +- #15833 remove out-of-date comment on pay-to-witness support (r8921039) +- #15821 Remove upgrade note in release notes from EOL versions (MarcoFalke) +- #15267 explain AcceptToMemoryPoolWorker's coins_to_uncache (jamesob) +- #15887 Align code example style with clang-format (hebasto) +- #15877 Fix -dustrelayfee= argument docs grammar (keepkeyjon) +- #15908 Align MSVC build options with Linux build ones (hebasto) +- #15941 Add historical release notes for 0.18.0 (laanwj) +- #15794 Clarify PR guidelines w/re documentation (dongcarl) +- #15607 Release process updates (jonatack) +- #14364 Clarify -blocksdir usage (sangaman) +- #15777 Add doxygen comments for keypool classes (jnewbery) +- #15820 Add productivity notes for dummy rebases (dongcarl) +- #15922 Explain how to pass in non-fundamental types into functions (MarcoFalke) +- #16080 build/doc: update bitcoin_config.h packages, release process (jonatack) +- #16047 analyzepsbt description in doc/psbt.md (jonatack) +- #16039 add release note for 14954 (fanquake) +- #16139 Add riscv64 to outputs list in release-process.md (JeremyRand) +- #16140 create security policy (narula) +- #16164 update release process for SECURITY.md (jonatack) +- #16213 Remove explicit mention of versions from SECURITY.md (MarcoFalke) +- #16186 doc/lint: Fix spelling errors identified by codespell 1.15.0 (Empact) +- #16149 Rework section on ACK in CONTRIBUTING.md (MarcoFalke) +- #16196 Add release notes for 14897 & 15834 (MarcoFalke) +- #16241 add rapidcheck to vcpkg install list (fanquake) +- #16243 Remove travis badge from readme (MarcoFalke) +- #16256 remove orphaned header in developer notes (jonatack) +- #15964 Improve build-osx document formatting (giulio92) +- #16313 Fix broken link in doc/build-osx.md (jonatack) +- #16330 Use placeholder instead of key expiration date (hebasto) +- #16339 add reduce-memory.md (fanquake) +- #16347 Include static members in Doxygen (dongcarl) +- #15824 Improve netbase comments (dongcarl) +- #16430 Update bips 35, 37 and 111 status (MarcoFalke) +- #16455 Remove downgrading warning in release notes, per 0.18 branch (MarcoFalke) +- #16484 update labels in CONTRIBUTING.md (MarcoFalke) +- #16483 update Python command in msvc readme (sipsorcery) +- #16504 Add release note for the deprecated totalFee option of bumpfee (promag) +- #16448 add note on precedence of options in bitcoin.conf (fanquake) +- #16536 Update and extend benchmarking.md (ariard) +- #16530 Fix grammar and punctuation in developer notes (Tech1k) +- #16574 Add historical release notes for 0.18.1 (laanwj) +- #16585 Update Markdown syntax for bdb packages (emilengler) +- #16586 Mention other ways to conserve memory on compilation (MarcoFalke) +- #16605 Add missing contributor to 0.18.1 release notes (meshcollider) +- #16615 Fix typos in COPYRIGHT (gapeman) +- #16626 Fix spelling error chache -> cache (nilswloewen) +- #16587 Improve versionbits.h documentation (ariard) +- #16643 Add ZMQ dependencies to the Fedora build instructions (hebasto) +- #16634 Refer in rpcbind doc to the manpage (MarcoFalke) +- #16555 mention whitelist is inbound, and applies to blocksonly (Sjors) +- #16645 initial RapidCheck property-based testing documentation (jonatack) +- #16691 improve depends prefix documentation (fanquake) +- #16629 Add documentation for the new whitelist permissions (NicolasDorier) +- #16723 Update labels in CONTRIBUTING.md (hebasto) +- #16461 Tidy up shadowing section (promag) +- #16621 add default bitcoin.conf locations (GChuf) +- #16752 Delete stale URL in test README (michaelfolkson) +- #14862 Declare BLOCK_VALID_HEADER reserved (MarcoFalke) +- #16806 Add issue templates for bug and feature request (MarcoFalke) +- #16857 Elaborate need to re-login on Debian-based after usermod for Tor group (clashicly) +- #16863 Add a missing closing parenthesis in the bitcoin-wallet's help (darosior) +- #16757 CChainState return values (MarcoFalke) +- #16847 add comments clarifying how local services are advertised (jamesob) +- #16812 Fix whitespace errs in .md files, bitcoin.conf, and Info.plist.in (ch4ot1c) +- #16885 Update tx-size-small comment with relevant CVE disclosure (instagibbs) +- #16900 Fix doxygen comment for SignTransaction in rpc/rawtransaction_util (MarcoFalke) +- #16914 Update homebrew instruction for doxygen (Sjors) +- #16912 Remove Doxygen intro from src/bitcoind.cpp (ch4ot1c) +- #16960 replace outdated OpenSSL comment in test README (fanquake) +- #16968 Remove MSVC update step from translation process (laanwj) +- #16953 Improve test READMEs (fjahr) +- #16962 Put PR template in comments (laanwj) +- #16397 Clarify includeWatching for fundrawtransaction (stevenroose) +- #15459 add how to calculate blockchain and chainstate size variables to release process (marcoagner) +- #16997 Update bips.md for 0.19 (laanwj) +- #17001 Remove mention of renamed mapBlocksUnlinked (MarcoFalke) +- #17014 Consolidate release notes before 0.19.0 (move-only) (MarcoFalke) +- #17111 update bips.md with buried BIP9 deployments (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 251 +- Aaron Clauson +- Akio Nakamura +- Alistair Mann +- Amiti Uttarwar +- Andrew Chow +- andrewtoth +- Anthony Towns +- Antoine Riard +- Aseem Sood +- Ben Carman +- Ben Woosley +- bpay +- Carl Dong +- Carnhof Daki +- Chris Capobianco +- Chris Moore +- Chuf +- clashic +- clashicly +- Cory Fields +- Daki Carnhof +- Dan Gershony +- Daniel Edgecumbe +- Daniel Kraft +- Daniel McNally +- darosior +- David A. Harding +- David Reikher +- Douglas Roark +- Elichai Turkel +- Emil +- Emil Engler +- ezegom +- Fabian Jahr +- fanquake +- Felix Weis +- Ferdinando M. Ametrano +- fridokus +- gapeman +- GChuf +- Gert-Jaap Glasbergen +- Giulio Lombardo +- Glenn Willen +- Graham Krizek +- Gregory Sanders +- grim-trigger +- gwillen +- Hennadii Stepanov +- Jack Mallers +- James Hilliard +- James O'Beirne +- Jan Beich +- Jeremy Rubin +- JeremyRand +- Jim Posen +- John Bampton +- John Newbery +- Jon Atack +- Jon Layton +- Jonas Schnelli +- Jonathan "Duke" Leto +- João Barbosa +- Joonmo Yang +- Jordan Baczuk +- Jorge Timón +- Josu Goñi +- Julian Fleischer +- Karl-Johan Alm +- Kaz Wesley +- keepkeyjon +- Kirill Fomichev +- Kristaps Kaupe +- Kristian Kramer +- Larry Ruane +- Lenny Maiorani +- LongShao007 +- Luca Venturini +- lucash-dev +- Luke Dashjr +- marcoagner +- MarcoFalke +- marcuswin +- Martin Ankerl +- Martin Zumsande +- Matt Corallo +- MeshCollider +- Michael Folkson +- Miguel Herranz +- Nathan Marley +- Neha Narula +- nicolas.dorier +- Nils Loewen +- nkostoulas +- NullFunctor +- orient +- Patrick Strateman +- Peter Bushnell +- Peter Wagner +- Pieter Wuille +- practicalswift +- qmma +- r8921039 +- RJ Rybarczyk +- Russell Yanofsky +- Samuel Dobson +- Sebastian Falbesoner +- setpill +- shannon1916 +- Sjors Provoost +- soroosh-sdi +- Steven Roose +- Suhas Daftuar +- tecnovert +- THETCR +- Tim Ruffing +- Tobias Kaderle +- Torkel Rogstad +- Ulrich Kempken +- whythat +- William Casarin +- Wladimir J. van der Laan +- zenosage + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-0.19.0.md b/doc/release-notes/release-notes-0.19.0.md new file mode 100644 index 0000000000..086491e92d --- /dev/null +++ b/doc/release-notes/release-notes-0.19.0.md @@ -0,0 +1,6 @@ +0.19.0 note +----------- + +Due to a last-minute issue (#17449), 0.19.0, although it was tagged, was never released. + +See the release notes for 0.19.0.1 instead. diff --git a/doc/release-process.md b/doc/release-process.md index 2c3c4e3869..1ffef3e106 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -44,7 +44,8 @@ Release Process #### After branch-off (on the major release branch) -- Update the versions and the link to the release notes draft in `doc/release-notes.md`. +- Update the versions. +- Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. #### Before final release @@ -116,10 +117,8 @@ Ensure gitian-builder is up-to-date: pushd ./gitian-builder mkdir -p inputs - wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch - echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c - wget -P inputs https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz - echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c + wget -O inputs/osslsigncode-2.0.tar.gz https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz + echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c popd Create the macOS SDK tarball, see the [macOS build instructions](build-osx.md#deterministic-macos-dmg-notes) for details, and copy it into the inputs directory. @@ -317,7 +316,7 @@ bitcoin.org (see below for bitcoin.org update instructions). instructions: https://github.com/bitcoin-dot-org/bitcoin.org/blob/master/docs/adding-events-release-notes-and-alerts.md#release-notes - After the pull request is merged, the website will automatically show the newest version within 15 minutes, as well - as update the OS download links. Ping @saivann/@harding (saivann/harding on Freenode) in case anything goes wrong + as update the OS download links. - Update other repositories and websites for new version @@ -332,7 +331,12 @@ bitcoin.org (see below for bitcoin.org update instructions). - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) - - Create a new branch for the major release "0.xx" (used to build the snap package) + - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 + + - Push the latest version to master (if applicable), e.g. https://github.com/bitcoin-core/packaging/pull/32 + + - Create a new branch for the major release "0.xx" from master (used to build the snap package) and request the + track (if applicable), e.g. https://forum.snapcraft.io/t/track-request-for-bitcoin-core-snap/10112/7 - Notify MarcoFalke so that he can start building the snap package @@ -356,8 +360,6 @@ bitcoin.org (see below for bitcoin.org update instructions). - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes - - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/15555) for an example) and provide a link to it in the release announcements where useful - - Announce the release: - bitcoin-dev and bitcoin-core-dev mailing list diff --git a/libbitcoinconsensus.pc.in b/libbitcoinconsensus.pc.in index eb920c47eb..1ceab280bb 100644 --- a/libbitcoinconsensus.pc.in +++ b/libbitcoinconsensus.pc.in @@ -8,4 +8,3 @@ Description: Library for the Bitcoin consensus protocol. Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lbitcoinconsensus Cflags: -I${includedir} -Requires.private: libcrypto diff --git a/share/genbuild.sh b/share/genbuild.sh index cd6214d5fa..197787d5e0 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index ffa001bb7f..275078c88d 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>10.10.0</string> + <string>10.12.0</string> <key>LSArchitecturePriority</key> <array> diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index 3cc0fa1b1d..4efc4bd6f5 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2018 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/share/setup.nsi.in b/share/setup.nsi.in index e9aa1f2b73..be482ae741 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -1,4 +1,4 @@ -Name "@PACKAGE_NAME@ (@WINDOWS_BITS@-bit)" +Name "@PACKAGE_NAME@ (64-bit)" RequestExecutionLevel highest SetCompressor /SOLID lzma @@ -28,9 +28,7 @@ SetCompressor /SOLID lzma # Included files !include Sections.nsh !include MUI2.nsh -!if "@WINDOWS_BITS@" == "64" !include x64.nsh -!endif # Variables Var StartMenuGroup @@ -48,12 +46,8 @@ Var StartMenuGroup !insertmacro MUI_LANGUAGE English # Installer attributes -OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win@WINDOWS_BITS@-setup.exe -!if "@WINDOWS_BITS@" == "64" +OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win64-setup-unsigned.exe InstallDir $PROGRAMFILES64\Bitcoin -!else -InstallDir $PROGRAMFILES\Bitcoin -!endif CRCCheck on XPStyle on BrandingText " " @@ -94,7 +88,7 @@ Section -post SEC0001 !insertmacro MUI_STARTMENU_WRITE_BEGIN Application CreateDirectory $SMPROGRAMS\$StartMenuGroup CreateShortcut "$SMPROGRAMS\$StartMenuGroup\$(^Name).lnk" $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ - CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, @WINDOWS_BITS@-bit).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 1 + CreateShortcut "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, 64-bit).lnk" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" "-testnet" "$INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@" 1 CreateShortcut "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" $INSTDIR\uninstall.exe !insertmacro MUI_STARTMENU_WRITE_END WriteRegStr HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" DisplayName "$(^Name)" @@ -138,7 +132,7 @@ Section -un.post UNSEC0001 DeleteRegKey HKCU "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$(^Name)" Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\Uninstall $(^Name).lnk" Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\$(^Name).lnk" - Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, @WINDOWS_BITS@-bit).lnk" + Delete /REBOOTOK "$SMPROGRAMS\$StartMenuGroup\@PACKAGE_NAME@ (testnet, 64-bit).lnk" Delete /REBOOTOK "$SMSTARTUP\Bitcoin.lnk" Delete /REBOOTOK $INSTDIR\uninstall.exe Delete /REBOOTOK $INSTDIR\debug.log @@ -160,7 +154,6 @@ SectionEnd # Installer functions Function .onInit InitPluginsDir -!if "@WINDOWS_BITS@" == "64" ${If} ${RunningX64} ; disable registry redirection (enable access to 64-bit portion of registry) SetRegView 64 @@ -168,7 +161,6 @@ Function .onInit MessageBox MB_OK|MB_ICONSTOP "Cannot install 64-bit version on a 32-bit system." Abort ${EndIf} -!endif FunctionEnd # Uninstaller functions diff --git a/src/Makefile.am b/src/Makefile.am index 2093ed15e2..e58a89ca03 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,7 +19,7 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) @@ -119,6 +119,7 @@ BITCOIN_CORE_H = \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ + compat/cpuid.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ @@ -146,19 +147,23 @@ BITCOIN_CORE_H = \ dbwrapper.h \ limitedmap.h \ logging.h \ + logging/timer.h \ memusage.h \ merkleblock.h \ miner.h \ net.h \ net_permissions.h \ net_processing.h \ + net_types.h \ netaddress.h \ netbase.h \ netmessagemaker.h \ node/coin.h \ node/coinstats.h \ + node/context.h \ node/psbt.h \ node/transaction.h \ + node/utxo_snapshot.h \ noui.h \ optional.h \ outputtype.h \ @@ -171,6 +176,7 @@ BITCOIN_CORE_H = \ protocol.h \ psbt.h \ random.h \ + randomenv.h \ reverse_iterator.h \ reverselock.h \ rpc/blockchain.h \ @@ -204,8 +210,10 @@ BITCOIN_CORE_H = \ txmempool.h \ ui_interface.h \ undo.h \ + util/asmap.h \ util/bip32.h \ util/bytevectorhash.h \ + util/check.h \ util/error.h \ util/fees.h \ util/spanparsing.h \ @@ -214,6 +222,7 @@ BITCOIN_CORE_H = \ util/memory.h \ util/moneystr.h \ util/rbf.h \ + util/settings.h \ util/string.h \ util/threadnames.h \ util/time.h \ @@ -235,6 +244,7 @@ BITCOIN_CORE_H = \ wallet/load.h \ wallet/psbtwallet.h \ wallet/rpcwallet.h \ + wallet/scriptpubkeyman.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/wallettool.h \ @@ -283,6 +293,7 @@ libbitcoin_server_a_SOURCES = \ net_processing.cpp \ node/coin.cpp \ node/coinstats.cpp \ + node/context.cpp \ node/psbt.cpp \ node/transaction.cpp \ noui.cpp \ @@ -338,11 +349,11 @@ libbitcoin_wallet_a_SOURCES = \ wallet/db.cpp \ wallet/feebumper.cpp \ wallet/fees.cpp \ - wallet/ismine.cpp \ wallet/load.cpp \ wallet/psbtwallet.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ + wallet/scriptpubkeyman.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ wallet/walletutil.cpp \ @@ -495,10 +506,12 @@ libbitcoin_util_a_SOURCES = \ interfaces/handler.cpp \ logging.cpp \ random.cpp \ + randomenv.cpp \ rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ + util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ util/error.cpp \ @@ -506,6 +519,7 @@ libbitcoin_util_a_SOURCES = \ util/system.cpp \ util/moneystr.cpp \ util/rbf.cpp \ + util/settings.cpp \ util/threadnames.cpp \ util/spanparsing.cpp \ util/strencodings.cpp \ @@ -542,12 +556,9 @@ if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif -# Libraries below may be listed more than once to resolve circular dependencies (see -# https://eli.thegreenplace.net/2013/07/09/library-order-in-static-linking#circular-dependency) bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ @@ -559,7 +570,7 @@ bitcoind_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) +bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp @@ -577,7 +588,7 @@ bitcoin_cli_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) -bitcoin_cli_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) +bitcoin_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) # # bitcoin-tx binary # @@ -598,7 +609,7 @@ bitcoin_tx_LDADD = \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) -bitcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +bitcoin_tx_LDADD += $(BOOST_LIBS) # # bitcoin-wallet binary # @@ -625,7 +636,7 @@ bitcoin_wallet_LDADD = \ $(LIBSECP256K1) \ $(LIBUNIVALUE) -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) +bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # @@ -690,21 +701,28 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) +if TARGET_DARWIN + @echo "Checking macOS dynamic libraries..." + $(AM_V_at) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) +endif + if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." - $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif if EMBEDDED_LEVELDB include Makefile.leveldb.include endif +include Makefile.test_util.include + if ENABLE_TESTS include Makefile.test.include endif diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 38143e32b9..1c97e22de8 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -30,6 +30,7 @@ bench_bench_bitcoin_SOURCES = \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/mempool_stress.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ @@ -38,13 +39,7 @@ bench_bench_bitcoin_SOURCES = \ bench/bech32.cpp \ bench/lockedpool.cpp \ bench/poly1305.cpp \ - bench/prevector.cpp \ - test/lib/transaction_utils.h \ - test/lib/transaction_utils.cpp \ - test/setup_common.h \ - test/setup_common.cpp \ - test/util.h \ - test/util.cpp + bench/prevector.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) @@ -58,6 +53,7 @@ bench_bench_bitcoin_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ + $(LIBTEST_UTIL) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ @@ -75,7 +71,7 @@ bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 13b1470b58..cf09eee2cb 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -136,6 +136,7 @@ BITCOIN_QT_H = \ qt/rpcconsole.h \ qt/sendcoinsdialog.h \ qt/sendcoinsentry.h \ + qt/sendcoinsrecipient.h \ qt/signverifymessagedialog.h \ qt/splashscreen.h \ qt/trafficgraphwidget.h \ @@ -314,7 +315,6 @@ endif qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_bitcoin_qt_LDADD += $(CRYPTO_LIBS) qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_bitcoin_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index fad4873545..79db5cd7b4 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -10,6 +10,7 @@ QT_TS = \ qt/locale/bitcoin_de_DE.ts \ qt/locale/bitcoin_el.ts \ qt/locale/bitcoin_el_GR.ts \ + qt/locale/bitcoin_en.ts \ qt/locale/bitcoin_en_AU.ts \ qt/locale/bitcoin_en_GB.ts \ qt/locale/bitcoin_eo.ts \ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index c309340fd7..8c47fabad9 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -26,12 +26,6 @@ TEST_QT_H = \ qt/test/util.h \ qt/test/wallettests.h -TEST_BITCOIN_CPP = \ - test/setup_common.cpp - -TEST_BITCOIN_H = \ - test/setup_common.h - qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_TEST_INCLUDES) @@ -42,9 +36,7 @@ qt_test_test_bitcoin_qt_SOURCES = \ qt/test/test_main.cpp \ qt/test/uritests.cpp \ qt/test/util.cpp \ - $(TEST_QT_H) \ - $(TEST_BITCOIN_CPP) \ - $(TEST_BITCOIN_H) + $(TEST_QT_H) if ENABLE_WALLET qt_test_test_bitcoin_qt_SOURCES += \ qt/test/addressbooktests.cpp \ @@ -54,7 +46,7 @@ endif # ENABLE_WALLET nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) -qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) +qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) $(LIBTEST_UTIL) if ENABLE_WALLET qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif @@ -63,7 +55,7 @@ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ - $(QR_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 019e832cc6..9d782e7a04 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -4,10 +4,17 @@ FUZZ_TARGETS = \ + test/fuzz/addr_info_deserialize \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ test/fuzz/banentry_deserialize \ + test/fuzz/base_encode_decode \ + test/fuzz/bech32 \ + test/fuzz/block \ test/fuzz/block_deserialize \ + test/fuzz/block_file_info_deserialize \ + test/fuzz/block_filter_deserialize \ + test/fuzz/block_header_and_short_txids_deserialize \ test/fuzz/blockheader_deserialize \ test/fuzz/blocklocator_deserialize \ test/fuzz/blockmerkleroot \ @@ -16,17 +23,43 @@ FUZZ_TARGETS = \ test/fuzz/blockundo_deserialize \ test/fuzz/bloomfilter_deserialize \ test/fuzz/coins_deserialize \ + test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ + test/fuzz/fee_rate_deserialize \ + test/fuzz/flat_file_pos_deserialize \ + test/fuzz/hex \ + test/fuzz/integer \ test/fuzz/inv_deserialize \ + test/fuzz/key_origin_info_deserialize \ + test/fuzz/merkle_block_deserialize \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ + test/fuzz/out_point_deserialize \ + test/fuzz/parse_hd_keypath \ + test/fuzz/parse_iso8601 \ + test/fuzz/parse_numbers \ + test/fuzz/parse_script \ + test/fuzz/parse_univalue \ + test/fuzz/partial_merkle_tree_deserialize \ + test/fuzz/partially_signed_transaction_deserialize \ + test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/psbt \ + test/fuzz/psbt_input_deserialize \ + test/fuzz/psbt_output_deserialize \ + test/fuzz/pub_key_deserialize \ test/fuzz/script \ + test/fuzz/script_deserialize \ test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/spanparsing \ + test/fuzz/strprintf \ + test/fuzz/sub_net_deserialize \ test/fuzz/transaction \ + test/fuzz/tx_in \ + test/fuzz/tx_in_deserialize \ + test/fuzz/tx_out \ test/fuzz/txoutcompressor_deserialize \ test/fuzz/txundo_deserialize @@ -50,20 +83,16 @@ JSON_TEST_FILES = \ test/data/tx_invalid.json \ test/data/tx_valid.json -RAW_TEST_FILES = +RAW_TEST_FILES = \ + test/data/asmap.raw GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) BITCOIN_TEST_SUITE = \ - test/lib/transaction_utils.h \ - test/lib/transaction_utils.cpp \ test/main.cpp \ - test/setup_common.h \ - test/setup_common.cpp + $(TEST_UTIL_H) FUZZ_SUITE = \ - test/setup_common.h \ - test/setup_common.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h @@ -72,8 +101,10 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ + $(LIBTEST_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ @@ -81,7 +112,6 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(EVENT_LIBS) \ - $(CRYPTO_LIBS) \ $(EVENT_PTHREADS_LIBS) # test_bitcoin binary # @@ -117,6 +147,7 @@ BITCOIN_TESTS =\ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/logging_tests.cpp \ test/dbwrapper_tests.cpp \ test/validation_tests.cpp \ test/mempool_tests.cpp \ @@ -141,6 +172,7 @@ BITCOIN_TESTS =\ test/script_standard_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ + test/settings_tests.cpp \ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ @@ -156,6 +188,7 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_flush_tests.cpp \ test/versionbits_tests.cpp if ENABLE_PROPERTY_TESTS @@ -186,7 +219,7 @@ endif test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) -test_test_bitcoin_LDADD = +test_test_bitcoin_LDADD = $(LIBTEST_UTIL) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif @@ -195,7 +228,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_C $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_bitcoin_LDADD += $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(RAPIDCHECK_LIBS) +test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(RAPIDCHECK_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ @@ -203,155 +236,354 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ -test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 +test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addr_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addr_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 +test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 +test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 +test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_base_encode_decode_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_base_encode_decode_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_base_encode_decode_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_base_encode_decode_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_base_encode_decode_SOURCES = $(FUZZ_SUITE) test/fuzz/base_encode_decode.cpp + +test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bech32_SOURCES = $(FUZZ_SUITE) test/fuzz/bech32.cpp + +test_fuzz_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_SOURCES = $(FUZZ_SUITE) test/fuzz/block.cpp + test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_block_file_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILE_INFO_DESERIALIZE=1 +test_fuzz_block_file_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_file_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_file_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_file_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_block_filter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILTER_DESERIALIZE=1 +test_fuzz_block_filter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_filter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_filter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_filter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_block_header_and_short_txids_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE=1 +test_fuzz_block_header_and_short_txids_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_header_and_short_txids_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_header_and_short_txids_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 +test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 -test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 -test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 -test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 +test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 -test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 +test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 +test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_decode_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_decode_tx_SOURCES = $(FUZZ_SUITE) test/fuzz/decode_tx.cpp -test_fuzz_descriptor_parse_SOURCES = $(FUZZ_SUITE) test/fuzz/descriptor_parse.cpp test_fuzz_descriptor_parse_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_descriptor_parse_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_descriptor_parse_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_descriptor_parse_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_descriptor_parse_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_descriptor_parse_SOURCES = $(FUZZ_SUITE) test/fuzz/descriptor_parse.cpp + +test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 +test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_eval_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_eval_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_eval_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_eval_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_eval_script_SOURCES = $(FUZZ_SUITE) test/fuzz/eval_script.cpp + +test_fuzz_fee_rate_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFEE_RATE_DESERIALIZE=1 +test_fuzz_fee_rate_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fee_rate_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 +test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_flat_file_pos_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_hex_SOURCES = $(FUZZ_SUITE) test/fuzz/hex.cpp + +test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_integer_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_integer_SOURCES = $(FUZZ_SUITE) test/fuzz/integer.cpp + +test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 +test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_key_origin_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DKEY_ORIGIN_INFO_DESERIALIZE=1 +test_fuzz_key_origin_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_key_origin_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_merkle_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMERKLE_BLOCK_DESERIALIZE=1 +test_fuzz_merkle_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_merkle_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_merkle_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_merkle_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 +test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_out_point_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DOUT_POINT_DESERIALIZE=1 +test_fuzz_out_point_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_out_point_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_out_point_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_out_point_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_parse_hd_keypath_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_hd_keypath_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_hd_keypath_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_hd_keypath_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_hd_keypath_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_hd_keypath.cpp + +test_fuzz_parse_iso8601_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_iso8601_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_iso8601_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_iso8601.cpp + +test_fuzz_parse_numbers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_numbers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_numbers_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_numbers_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_numbers_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_numbers.cpp + +test_fuzz_parse_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_script_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_script.cpp + +test_fuzz_parse_univalue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_univalue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_univalue_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_univalue.cpp + +test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 +test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partial_merkle_tree_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_partially_signed_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIALLY_SIGNED_TRANSACTION_DESERIALIZE=1 +test_fuzz_partially_signed_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partially_signed_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_prefilled_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPREFILLED_TRANSACTION_DESERIALIZE=1 +test_fuzz_prefilled_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prefilled_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_psbt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_psbt_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_SOURCES = $(FUZZ_SUITE) test/fuzz/psbt.cpp + +test_fuzz_psbt_input_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_INPUT_DESERIALIZE=1 +test_fuzz_psbt_input_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_input_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_input_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_input_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_psbt_output_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_OUTPUT_DESERIALIZE=1 +test_fuzz_psbt_output_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_output_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_output_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_output_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_pub_key_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPUB_KEY_DESERIALIZE=1 +test_fuzz_pub_key_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_pub_key_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_pub_key_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_pub_key_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_script_SOURCES = $(FUZZ_SUITE) test/fuzz/script.cpp test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_SOURCES = $(FUZZ_SUITE) test/fuzz/script.cpp + +test_fuzz_script_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSCRIPT_DESERIALIZE=1 +test_fuzz_script_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp -test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_spanparsing_SOURCES = $(FUZZ_SUITE) test/fuzz/spanparsing.cpp test_fuzz_spanparsing_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_spanparsing_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_spanparsing_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_spanparsing_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_spanparsing_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_spanparsing_SOURCES = $(FUZZ_SUITE) test/fuzz/spanparsing.cpp -test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 -test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_strprintf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_strprintf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_strprintf_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_strprintf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_strprintf_SOURCES = $(FUZZ_SUITE) test/fuzz/strprintf.cpp -test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 -test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_sub_net_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSUB_NET_DESERIALIZE=1 +test_fuzz_sub_net_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_sub_net_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 -test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_transaction_SOURCES = $(FUZZ_SUITE) test/fuzz/transaction.cpp -test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 -test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_in_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_tx_in_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_tx_in_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_in_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_SOURCES = $(FUZZ_SUITE) test/fuzz/tx_in.cpp -test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 -test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_in_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTX_IN_DESERIALIZE=1 +test_fuzz_tx_in_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_tx_in_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_in_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_eval_script_SOURCES = $(FUZZ_SUITE) test/fuzz/eval_script.cpp -test_fuzz_eval_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_eval_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_eval_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_eval_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_out_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_tx_out_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_tx_out_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_out_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_out_SOURCES = $(FUZZ_SUITE) test/fuzz/tx_out.cpp -test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 -test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 -test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_transaction_SOURCES = $(FUZZ_SUITE) test/fuzz/transaction.cpp -test_fuzz_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 +test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp endif # ENABLE_FUZZ @@ -411,3 +643,12 @@ endif echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" + +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include new file mode 100644 index 0000000000..505d630b7d --- /dev/null +++ b/src/Makefile.test_util.include @@ -0,0 +1,34 @@ +# Copyright (c) 2013-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +LIBTEST_UTIL=libtest_util.a + +EXTRA_LIBRARIES += \ + $(LIBTEST_UTIL) + +TEST_UTIL_H = \ + test/util/blockfilter.h \ + test/util/logging.h \ + test/util/mining.h \ + test/util/setup_common.h \ + test/util/str.h \ + test/util/transaction_utils.h \ + test/util/wallet.h + +libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +libtest_util_a_SOURCES = \ + test/util/blockfilter.cpp \ + test/util/logging.cpp \ + test/util/mining.cpp \ + test/util/setup_common.cpp \ + test/util/str.cpp \ + test/util/transaction_utils.cpp \ + test/util/wallet.cpp \ + $(TEST_UTIL_H) + +LIBTEST_UTIL += $(LIBBITCOIN_SERVER) +LIBTEST_UTIL += $(LIBBITCOIN_COMMON) +LIBTEST_UTIL += $(LIBBITCOIN_UTIL) +LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index db936486b6..835c5d6c65 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/addrdb.h b/src/addrdb.h index 290b63dd12..c6d4307d69 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #define BITCOIN_ADDRDB_H #include <fs.h> +#include <net_types.h> // For banmap_t #include <serialize.h> #include <string> @@ -48,15 +49,7 @@ public: banReason = ban_reason_in; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(nCreateTime); - READWRITE(nBanUntil); - READWRITE(banReason); - } + SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); } void SetNull() { @@ -79,8 +72,6 @@ public: } }; -typedef std::map<CSubNet, CBanEntry> banmap_t; - /** Access to the (IP) address database (peers.dat) */ class CAddrDB { diff --git a/src/addrman.cpp b/src/addrman.cpp index 32676f8fa5..121ae4bf7e 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,20 +7,27 @@ #include <hash.h> #include <serialize.h> +#include <logging.h> -int CAddrInfo::GetTriedBucket(const uint256& nKey) const +int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i.\n", ToStringIP(), mapped_as, tried_bucket); + return tried_bucket; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const +int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const { - std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetCheapHash(); + std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - return hash2 % ADDRMAN_NEW_BUCKET_COUNT; + int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i.\n", ToStringIP(), mapped_as, new_bucket); + return new_bucket; } int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const @@ -153,7 +160,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -169,7 +176,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey); + int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -233,7 +240,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey); + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -301,7 +308,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; @@ -439,7 +446,7 @@ int CAddrMan::Check_() if (vvTried[n][i] != -1) { if (!setTried.count(vvTried[n][i])) return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n) + if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) return -17; if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) return -18; @@ -545,7 +552,7 @@ void CAddrMan::ResolveCollisions_() CAddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey); + int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -609,10 +616,33 @@ CAddrInfo CAddrMan::SelectTriedCollision_() CAddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); int id_old = vvTried[tried_bucket][tried_bucket_pos]; return mapInfo[id_old]; } + +std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) +{ + std::vector<bool> bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk.\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + fseek(filestr, 0, SEEK_SET); + char cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + return bits; +} diff --git a/src/addrman.h b/src/addrman.h index e54184ce35..5901611bee 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,11 +12,17 @@ #include <sync.h> #include <timedata.h> #include <util/system.h> +#include <clientversion.h> #include <map> #include <set> #include <stdint.h> #include <vector> +#include <iostream> +#include <streams.h> +#include <fs.h> +#include <hash.h> + /** * Extended statistics about a CAddress @@ -53,14 +59,10 @@ private: public: - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CAddress, *this); - READWRITE(source); - READWRITE(nLastSuccess); - READWRITE(nAttempts); + SERIALIZE_METHODS(CAddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); } CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) @@ -72,15 +74,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey) const; + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src) const; + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey) const + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const { - return GetNewBucket(nKey, source); + return GetNewBucket(nKey, source, asmap); } //! Calculate in which position of a bucket to store this entry. @@ -174,9 +176,10 @@ static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes */ class CAddrMan { +friend class CAddrManTest; protected: //! critical section to protect the inner data structures - mutable CCriticalSection cs; + mutable RecursiveMutex cs; private: //! last used nId @@ -268,9 +271,29 @@ protected: void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + std::vector<bool> m_asmap; + + // Read asmap from provided binary file + static std::vector<bool> DecodeAsmap(fs::path path); + + /** * serialized format: - * * version byte (currently 1) + * * version byte (1 for pre-asmap files, 2 for files including asmap version) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -294,7 +317,7 @@ public: * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports * changes to the ADDRMAN_ parameters without breaking the on-disk structure. * - * We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has + * We don't use SERIALIZE_METHODS since the serialization and deserialization code has * very little in common. */ template<typename Stream> @@ -302,7 +325,7 @@ public: { LOCK(cs); - unsigned char nVersion = 1; + unsigned char nVersion = 2; s << nVersion; s << ((unsigned char)32); s << nKey; @@ -345,6 +368,13 @@ public: } } } + // Store asmap version after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_version; + if (m_asmap.size() != 0) { + asmap_version = SerializeHash(m_asmap); + } + s << asmap_version; } template<typename Stream> @@ -353,7 +383,6 @@ public: LOCK(cs); Clear(); - unsigned char nVersion; s >> nVersion; unsigned char nKeySize; @@ -383,16 +412,6 @@ public: mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); - if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) { - // In case the new table data cannot be used (nVersion unknown, or bucket count wrong), - // immediately try to give them a reference based on their primary source address. - int nUBucket = info.GetNewBucket(nKey); - int nUBucketPos = info.GetBucketPosition(nKey, true, nUBucket); - if (vvNew[nUBucket][nUBucketPos] == -1) { - vvNew[nUBucket][nUBucketPos] = n; - info.nRefCount++; - } - } } nIdCount = nNew; @@ -401,7 +420,7 @@ public: for (int n = 0; n < nTried; n++) { CAddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (vvTried[nKBucket][nKBucketPos] == -1) { info.nRandomPos = vRandom.size(); @@ -417,7 +436,9 @@ public: } nTried -= nLost; - // Deserialize positions in the new table (if possible). + // Store positions in the new table buckets to apply later (if possible). + std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing + for (int bucket = 0; bucket < nUBuckets; bucket++) { int nSize = 0; s >> nSize; @@ -425,12 +446,38 @@ public: int nIndex = 0; s >> nIndex; if (nIndex >= 0 && nIndex < nNew) { - CAddrInfo &info = mapInfo[nIndex]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 1 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS) { - info.nRefCount++; - vvNew[bucket][nUBucketPos] = nIndex; - } + entryToBucket[nIndex] = bucket; + } + } + } + + uint256 supplied_asmap_version; + if (m_asmap.size() != 0) { + supplied_asmap_version = SerializeHash(m_asmap); + } + uint256 serialized_asmap_version; + if (nVersion > 1) { + s >> serialized_asmap_version; + } + + for (int n = 0; n < nNew; n++) { + CAddrInfo &info = mapInfo[n]; + int bucket = entryToBucket[n]; + int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; + } else { + // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + bucket = info.GetNewBucket(nKey, m_asmap); + nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][nUBucketPos] == -1) { + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; } } } diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 1111f27771..0bebb0cf54 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 171135b01f..a0a0429c2a 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/banman.cpp b/src/banman.cpp index 37fca7dd82..9cc584f0e4 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/banman.h b/src/banman.h index a1a00309dd..8984874914 100644 --- a/src/banman.h +++ b/src/banman.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BANMAN_H @@ -10,6 +10,7 @@ #include <addrdb.h> #include <fs.h> +#include <net_types.h> // For banmap_t #include <sync.h> // NOTE: When adjusting this, update rpcnet:setban's help ("24h") @@ -58,7 +59,7 @@ private: //!clean unused entries (if bantime has expired) void SweepBanned(); - CCriticalSection m_cs_banned; + RecursiveMutex m_cs_banned; banmap_t m_banned GUARDED_BY(m_cs_banned); bool m_is_dirty GUARDED_BY(m_cs_banned); CClientUIInterface* m_client_interface = nullptr; @@ -66,5 +67,4 @@ private: const int64_t m_default_ban_time; }; -extern std::unique_ptr<BanMan> g_banman; #endif diff --git a/src/base58.cpp b/src/base58.cpp index e3d2853399..6a9e21ffc2 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,10 +7,13 @@ #include <hash.h> #include <uint256.h> #include <util/strencodings.h> +#include <util/string.h> #include <assert.h> #include <string.h> +#include <limits> + /** All alphanumeric characters except for "0", "I", "O", and "l" */ static const char* pszBase58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; static const int8_t mapBase58[256] = { @@ -32,7 +35,7 @@ static const int8_t mapBase58[256] = { -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, }; -bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) +bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch, int max_ret_len) { // Skip leading spaces. while (*psz && IsSpace(*psz)) @@ -42,6 +45,7 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) int length = 0; while (*psz == '1') { zeroes++; + if (zeroes > max_ret_len) return false; psz++; } // Allocate enough space in big-endian base256 representation. @@ -62,6 +66,7 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) } assert(carry == 0); length = i; + if (length + zeroes > max_ret_len) return false; psz++; } // Skip trailing spaces. @@ -71,8 +76,6 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch) return false; // Skip leading zeroes in b256. std::vector<unsigned char>::iterator it = b256.begin() + (size - length); - while (it != b256.end() && *it == 0) - it++; // Copy result into output vector. vch.reserve(zeroes + (b256.end() - it)); vch.assign(zeroes, 0x00); @@ -126,9 +129,12 @@ std::string EncodeBase58(const std::vector<unsigned char>& vch) return EncodeBase58(vch.data(), vch.data() + vch.size()); } -bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet) +bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len) { - return DecodeBase58(str.c_str(), vchRet); + if (!ValidAsCString(str)) { + return false; + } + return DecodeBase58(str.c_str(), vchRet, max_ret_len); } std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn) @@ -140,9 +146,9 @@ std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn) return EncodeBase58(vch); } -bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet) +bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len) { - if (!DecodeBase58(psz, vchRet) || + if (!DecodeBase58(psz, vchRet, max_ret_len > std::numeric_limits<int>::max() - 4 ? std::numeric_limits<int>::max() : max_ret_len + 4) || (vchRet.size() < 4)) { vchRet.clear(); return false; @@ -157,7 +163,10 @@ bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet) return true; } -bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet) +bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret) { - return DecodeBase58Check(str.c_str(), vchRet); + if (!ValidAsCString(str)) { + return false; + } + return DecodeBase58Check(str.c_str(), vchRet, max_ret); } diff --git a/src/base58.h b/src/base58.h index d6e0299a1e..042ad671d3 100644 --- a/src/base58.h +++ b/src/base58.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,13 +35,13 @@ std::string EncodeBase58(const std::vector<unsigned char>& vch); * return true if decoding is successful. * psz cannot be nullptr. */ -NODISCARD bool DecodeBase58(const char* psz, std::vector<unsigned char>& vchRet); +NODISCARD bool DecodeBase58(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len); /** * Decode a base58-encoded string (str) into a byte vector (vchRet). * return true if decoding is successful. */ -NODISCARD bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet); +NODISCARD bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); /** * Encode a byte vector into a base58-encoded string, including checksum @@ -52,12 +52,12 @@ std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn); * Decode a base58-encoded string (psz) that includes a checksum into a byte * vector (vchRet), return true if decoding is successful */ -NODISCARD bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet); +NODISCARD bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len); /** * Decode a base58-encoded string (str) that includes a checksum into a byte * vector (vchRet), return true if decoding is successful */ -NODISCARD bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet); +NODISCARD bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); #endif // BITCOIN_BASE58_H diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 40a7b5e320..0690483d50 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -47,7 +47,7 @@ static void Base58Decode(benchmark::State& state) const char* addr = "17VZNX1SN5NtKa8UQFxwQbFeFc3iqRYhem"; std::vector<unsigned char> vch; while (state.KeepRunning()) { - (void) DecodeBase58(addr, vch); + (void) DecodeBase58(addr, vch, 64); } } diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 80f13eeb3b..f2fc3999fe 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index f2b520e893..5cf7e43f4b 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,7 +5,7 @@ #include <bench/bench.h> #include <chainparams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <validation.h> #include <algorithm> @@ -15,6 +15,9 @@ #include <numeric> #include <regex> +const RegTestingSetup* g_testing_setup = nullptr; +const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; + void benchmark::ConsolePrinter::header() { std::cout << "# Benchmark, evals, iterations, total, min, max, median" << std::endl; @@ -112,7 +115,9 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double printer.header(); for (const auto& p : benchmarks()) { - TestingSetup test{CBaseChainParams::REGTEST}; + RegTestingSetup test{}; + assert(g_testing_setup == nullptr); + g_testing_setup = &test; { LOCK(cs_main); assert(::ChainActive().Height() == 0); @@ -121,6 +126,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double } if (!std::regex_match(p.first, baseMatch, reFilter)) { + g_testing_setup = nullptr; continue; } @@ -133,6 +139,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double p.second.func(state); } printer.result(state); + g_testing_setup = nullptr; } printer.footer(); diff --git a/src/bench/bench.h b/src/bench/bench.h index 3a8c487b9a..6b7a0f76d1 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,9 @@ #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> +struct RegTestingSetup; +extern const RegTestingSetup* g_testing_setup; //!< A pointer to the current testing setup + // Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark // framework (see https://github.com/google/benchmark) // Why not use the Google Benchmark framework? Because adding Yet Another Dependency diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index d0d7c03ee1..9235d5fe6a 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -36,7 +36,7 @@ int main(int argc, char** argv) SetupBenchArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } @@ -51,9 +51,16 @@ int main(int argc, char** argv) std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); bool is_list_only = gArgs.GetBoolArg("-list", false); + if (evaluations == 0) { + return EXIT_SUCCESS; + } else if (evaluations < 0) { + tfm::format(std::cerr, "Error parsing evaluations argument: %d\n", evaluations); + return EXIT_FAILURE; + } + double scaling_factor; if (!ParseDouble(scaling_str, &scaling_factor)) { - tfm::format(std::cerr, "Error parsing scaling factor as double: %s\n", scaling_str.c_str()); + tfm::format(std::cerr, "Error parsing scaling factor as double: %s\n", scaling_str); return EXIT_FAILURE; } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 157f936a95..a113a73828 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -5,7 +5,9 @@ #include <bench/bench.h> #include <consensus/validation.h> #include <crypto/sha256.h> -#include <test/util.h> +#include <test/util/mining.h> +#include <test/util/setup_common.h> +#include <test/util/wallet.h> #include <txmempool.h> #include <validation.h> @@ -28,7 +30,7 @@ static void AssembleBlock(benchmark::State& state) std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(SCRIPT_PUB)); + tx.vin.push_back(MineBlock(g_testing_setup->m_node, SCRIPT_PUB)); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, SCRIPT_PUB); if (NUM_BLOCKS - b >= COINBASE_MATURITY) @@ -38,14 +40,14 @@ static void AssembleBlock(benchmark::State& state) LOCK(::cs_main); // Required for ::AcceptToMemoryPool. for (const auto& txr : txs) { - CValidationState state; - bool ret{::AcceptToMemoryPool(::mempool, state, txr, nullptr /* pfMissingInputs */, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; + TxValidationState state; + bool ret{::AcceptToMemoryPool(::mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; assert(ret); } } while (state.KeepRunning()) { - PrepareBlock(SCRIPT_PUB); + PrepareBlock(g_testing_setup->m_node, SCRIPT_PUB); } } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 39cab092cf..c313029ea8 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 4b13381e16..55786126b3 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -42,7 +42,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); - CValidationState validationState; + BlockValidationState validationState; bool checked = CheckBlock(block, validationState, chainParams->GetConsensus()); assert(checked); } diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 000a0259bb..f5f96a0136 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f2ab03e20e..d6d5e67c5b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,9 +1,10 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/context.h> #include <wallet/coinselection.h> #include <wallet/wallet.h> @@ -28,8 +29,10 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<st // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CoinSelection(benchmark::State& state) { - auto chain = interfaces::MakeChain(); - const CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + NodeContext node; + auto chain = interfaces::MakeChain(node); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet.SetupLegacyScriptPubKeyMan(); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -60,8 +63,9 @@ static void CoinSelection(benchmark::State& state) } typedef std::set<CInputCoin> CoinSet; -static auto testChain = interfaces::MakeChain(); -static const CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); +static NodeContext testNode; +static auto testChain = interfaces::MakeChain(testNode); +static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp @@ -90,6 +94,7 @@ static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) static void BnBExhaustion(benchmark::State& state) { // Setup + testWallet.SetupLegacyScriptPubKeyMan(); std::vector<OutputGroup> utxo_pool; CoinSet selection; CAmount value_ret = 0; diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 674753c191..a9d4d78888 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 6cfa3750d6..a783370b4e 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -54,7 +54,7 @@ static void DuplicateInputs(benchmark::State& state) block.hashMerkleRoot = BlockMerkleRoot(block); while (state.KeepRunning()) { - CValidationState cvstate{}; + BlockValidationState cvstate{}; assert(!CheckBlock(block, cvstate, chainparams.GetConsensus(), false, false)); assert(cvstate.GetRejectReason() == "bad-txns-inputs-duplicate"); } diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index 3595249559..60a4fbf0ba 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index 0d9b123400..5d943810df 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index a2a21c673b..1c9c106666 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp new file mode 100644 index 0000000000..389e2c096f --- /dev/null +++ b/src/bench/mempool_stress.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <policy/policy.h> +#include <txmempool.h> + +#include <vector> + +static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + int64_t nTime = 0; + unsigned int nHeight = 1; + bool spendsCoinbase = false; + unsigned int sigOpCost = 4; + LockPoints lp; + pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, spendsCoinbase, sigOpCost, lp)); +} + +struct Available { + CTransactionRef ref; + size_t vin_left{0}; + size_t tx_count; + Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){} +}; + +static void ComplexMemPool(benchmark::State& state) +{ + FastRandomContext det_rand{true}; + std::vector<Available> available_coins; + std::vector<CTransactionRef> ordered_coins; + // Create some base transactions + size_t tx_counter = 1; + for (auto x = 0; x < 100; ++x) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << CScriptNum(tx_counter); + tx.vin[0].scriptWitness.stack.push_back(CScriptNum(x).getvch()); + tx.vout.resize(det_rand.randrange(10)+2); + for (auto& out : tx.vout) { + out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL; + out.nValue = 10 * COIN; + } + ordered_coins.emplace_back(MakeTransactionRef(tx)); + available_coins.emplace_back(ordered_coins.back(), tx_counter++); + } + for (auto x = 0; x < 800 && !available_coins.empty(); ++x) { + CMutableTransaction tx = CMutableTransaction(); + size_t n_ancestors = det_rand.randrange(10)+1; + for (size_t ancestor = 0; ancestor < n_ancestors && !available_coins.empty(); ++ancestor){ + size_t idx = det_rand.randrange(available_coins.size()); + Available coin = available_coins[idx]; + uint256 hash = coin.ref->GetHash(); + // biased towards taking just one ancestor, but maybe more + size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left); + for (size_t i = 0; i < n_to_take; ++i) { + tx.vin.emplace_back(); + tx.vin.back().prevout = COutPoint(hash, coin.vin_left++); + tx.vin.back().scriptSig = CScript() << coin.tx_count; + tx.vin.back().scriptWitness.stack.push_back(CScriptNum(coin.tx_count).getvch()); + } + if (coin.vin_left == coin.ref->vin.size()) { + coin = available_coins.back(); + available_coins.pop_back(); + } + tx.vout.resize(det_rand.randrange(10)+2); + for (auto& out : tx.vout) { + out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL; + out.nValue = 10 * COIN; + } + } + ordered_coins.emplace_back(MakeTransactionRef(tx)); + available_coins.emplace_back(ordered_coins.back(), tx_counter++); + } + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + while (state.KeepRunning()) { + for (auto& tx : ordered_coins) { + AddTx(tx, pool); + } + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); + pool.TrimToSize(GetVirtualTransactionSize(*ordered_coins.front())); + } +} + +BENCHMARK(ComplexMemPool, 1); diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index cffdb388f8..6cdb4ff0a7 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index c9947f192e..31e166cc27 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,7 +10,7 @@ #include <script/script.h> #include <script/standard.h> #include <streams.h> -#include <test/lib/transaction_utils.h> +#include <test/util/transaction_utils.h> #include <array> diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 313b5a3ba0..62568a9da5 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -4,8 +4,11 @@ #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/context.h> #include <optional.h> -#include <test/util.h> +#include <test/util/mining.h> +#include <test/util/setup_common.h> +#include <test/util/wallet.h> #include <validationinterface.h> #include <wallet/wallet.h> @@ -13,9 +16,11 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b { const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(); + NodeContext node; + std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); CWallet wallet{chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; { + wallet.SetupLegacyScriptPubKeyMan(); bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); wallet.handleNotifications(); @@ -26,8 +31,8 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { - generatetoaddress(address_mine.get_value_or(ADDRESS_WATCHONLY)); - generatetoaddress(ADDRESS_WATCHONLY); + generatetoaddress(g_testing_setup->m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); + generatetoaddress(g_testing_setup->m_node, ADDRESS_WATCHONLY); } SyncWithValidationInterfaceQueue(); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 93b7a7152c..c085095a2b 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -105,7 +105,7 @@ static int AppInitRPC(int argc, char* argv[]) SetupCliArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { @@ -119,7 +119,7 @@ static int AppInitRPC(int argc, char* argv[]) strUsage += "\n" + gArgs.GetHelpMessage(); } - tfm::format(std::cout, "%s", strUsage.c_str()); + tfm::format(std::cout, "%s", strUsage); if (argc < 2) { tfm::format(std::cerr, "Error: too few parameters\n"); return EXIT_FAILURE; @@ -127,11 +127,11 @@ static int AppInitRPC(int argc, char* argv[]) return EXIT_SUCCESS; } if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "").c_str()); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); return EXIT_FAILURE; } if (!gArgs.ReadConfigFiles(error, true)) { - tfm::format(std::cerr, "Error reading configuration file: %s\n", error.c_str()); + tfm::format(std::cerr, "Error reading configuration file: %s\n", error); return EXIT_FAILURE; } // Check for -chain, -testnet or -regtest parameter (BaseParams() calls are only valid after this clause) @@ -256,17 +256,16 @@ public: return batch[ID_BLOCKCHAININFO]; } result.pushKV("version", batch[ID_NETWORKINFO]["result"]["version"]); - result.pushKV("protocolversion", batch[ID_NETWORKINFO]["result"]["protocolversion"]); result.pushKV("blocks", batch[ID_BLOCKCHAININFO]["result"]["blocks"]); + result.pushKV("headers", batch[ID_BLOCKCHAININFO]["result"]["headers"]); + result.pushKV("verificationprogress", batch[ID_BLOCKCHAININFO]["result"]["verificationprogress"]); result.pushKV("timeoffset", batch[ID_NETWORKINFO]["result"]["timeoffset"]); result.pushKV("connections", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); - if (!batch[ID_WALLETINFO].isNull()) { - result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); + if (!batch[ID_WALLETINFO]["result"].isNull()) { result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); - result.pushKV("keypoololdest", batch[ID_WALLETINFO]["result"]["keypoololdest"]); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); @@ -366,7 +365,7 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co std::string endpoint = "/"; if (!gArgs.GetArgs("-rpcwallet").empty()) { std::string walletName = gArgs.GetArg("-rpcwallet", ""); - char *encodedURI = evhttp_uriencode(walletName.c_str(), walletName.size(), false); + char *encodedURI = evhttp_uriencode(walletName.data(), walletName.size(), false); if (encodedURI) { endpoint = "/wallet/"+ std::string(encodedURI); free(encodedURI); @@ -393,7 +392,7 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string().c_str())); + GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string())); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } @@ -541,7 +540,7 @@ static int CommandLineRPC(int argc, char *argv[]) } if (strPrint != "") { - tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint.c_str()); + tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint); } return nRet; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index cabea610f3..735f55fba7 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -83,7 +83,7 @@ static int AppInitRawTx(int argc, char* argv[]) SetupBitcoinTxArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } @@ -105,7 +105,7 @@ static int AppInitRawTx(int argc, char* argv[]) "\n"; strUsage += gArgs.GetHelpMessage(); - tfm::format(std::cout, "%s", strUsage.c_str()); + tfm::format(std::cout, "%s", strUsage); if (argc < 2) { tfm::format(std::cerr, "Error: too few parameters\n"); @@ -724,21 +724,21 @@ static void OutputTxJSON(const CTransaction& tx) TxToUniv(tx, uint256(), entry); std::string jsonOutput = entry.write(4); - tfm::format(std::cout, "%s\n", jsonOutput.c_str()); + tfm::format(std::cout, "%s\n", jsonOutput); } static void OutputTxHash(const CTransaction& tx) { std::string strHexHash = tx.GetHash().GetHex(); // the hex-encoded transaction hash (aka the transaction id) - tfm::format(std::cout, "%s\n", strHexHash.c_str()); + tfm::format(std::cout, "%s\n", strHexHash); } static void OutputTxHex(const CTransaction& tx) { std::string strHex = EncodeHexTx(tx); - tfm::format(std::cout, "%s\n", strHex.c_str()); + tfm::format(std::cout, "%s\n", strHex); } static void OutputTx(const CTransaction& tx) @@ -829,7 +829,7 @@ static int CommandLineRawTx(int argc, char* argv[]) } if (strPrint != "") { - tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint.c_str()); + tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint); } return nRet; } diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index eda4f8ce78..7f1a4a114b 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,19 +36,19 @@ static bool WalletAppInit(int argc, char* argv[]) SetupWalletToolArgs(); std::string error_message; if (!gArgs.ParseParameters(argc, argv, error_message)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } if (argc < 2 || HelpRequested(gArgs)) { std::string usage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n\n" + - "wallet-tool is an offline tool for creating and interacting with Bitcoin Core wallet files.\n" + - "By default wallet-tool will act on wallets in the default mainnet wallet directory in the datadir.\n" + + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n\n" + gArgs.GetHelpMessage(); - tfm::format(std::cout, "%s", usage.c_str()); + tfm::format(std::cout, "%s", usage); return false; } @@ -56,7 +56,7 @@ static bool WalletAppInit(int argc, char* argv[]) LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", gArgs.GetBoolArg("-debug", false)); if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "").c_str()); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); return false; } // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause) @@ -87,7 +87,7 @@ int main(int argc, char* argv[]) for(int i = 1; i < argc; ++i) { if (!IsSwitchChar(argv[i][0])) { if (!method.empty()) { - tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method.c_str(), argv[i]); + tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method, argv[i]); return EXIT_FAILURE; } method = argv[i]; diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index ddd6f8839c..4b5cea4849 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -12,6 +12,7 @@ #include <compat.h> #include <init.h> #include <interfaces/chain.h> +#include <node/context.h> #include <noui.h> #include <shutdown.h> #include <ui_interface.h> @@ -24,13 +25,13 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -static void WaitForShutdown() +static void WaitForShutdown(NodeContext& node) { while (!ShutdownRequested()) { MilliSleep(200); } - Interrupt(); + Interrupt(node); } ////////////////////////////////////////////////////////////////////////////// @@ -39,8 +40,8 @@ static void WaitForShutdown() // static bool AppInit(int argc, char* argv[]) { - InitInterfaces interfaces; - interfaces.chain = interfaces::MakeChain(); + NodeContext node; + node.chain = interfaces::MakeChain(node); bool fRet = false; @@ -70,7 +71,7 @@ static bool AppInit(int argc, char* argv[]) strUsage += "\n" + gArgs.GetHelpMessage(); } - tfm::format(std::cout, "%s", strUsage.c_str()); + tfm::format(std::cout, "%s", strUsage); return true; } @@ -142,7 +143,7 @@ static bool AppInit(int argc, char* argv[]) // If locking the data directory failed, exit immediately return false; } - fRet = AppInitMain(interfaces); + fRet = AppInitMain(node); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); @@ -152,11 +153,11 @@ static bool AppInit(int argc, char* argv[]) if (!fRet) { - Interrupt(); + Interrupt(node); } else { - WaitForShutdown(); + WaitForShutdown(node); } - Shutdown(interfaces); + Shutdown(node); return fRet; } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index f0fcf675eb..263d863cfa 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -197,13 +197,13 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< if (vtx_missing.size() != tx_missing_offset) return READ_STATUS_INVALID; - CValidationState state; + BlockValidationState state; if (!CheckBlock(block, state, Params().GetConsensus())) { // TODO: We really want to just check merkle tree manually here, // but that is expensive, and CheckBlock caches a block's // "checked-status" (in the CBlock?). CBlock should be able to // check its own merkle root and cache that check. - if (state.GetReason() == ValidationInvalidReason::BLOCK_MUTATED) + if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) return READ_STATUS_FAILED; // Possible Short ID collision return READ_STATUS_CHECKBLOCK_FAILED; } diff --git a/src/blockencodings.h b/src/blockencodings.h index 18a6e35f31..55ed8989bb 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 787390be31..7aff3be6e7 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -1,9 +1,10 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <mutex> #include <sstream> +#include <set> #include <blockfilter.h> #include <crypto/siphash.h> @@ -221,15 +222,14 @@ bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type return false; } -const std::vector<BlockFilterType>& AllBlockFilterTypes() +const std::set<BlockFilterType>& AllBlockFilterTypes() { - static std::vector<BlockFilterType> types; + static std::set<BlockFilterType> types; static std::once_flag flag; std::call_once(flag, []() { - types.reserve(g_filter_types.size()); for (auto entry : g_filter_types) { - types.push_back(entry.first); + types.insert(entry.first); } }); diff --git a/src/blockfilter.h b/src/blockfilter.h index 914b94fec1..ff8744b217 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #include <stdint.h> #include <string> +#include <set> #include <unordered_set> #include <vector> @@ -97,7 +98,7 @@ const std::string& BlockFilterTypeName(BlockFilterType filter_type); bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type); /** Get a list of known filter types. */ -const std::vector<BlockFilterType>& AllBlockFilterTypes(); +const std::set<BlockFilterType>& AllBlockFilterTypes(); /** Get a comma-separated list of known filter type names. */ const std::string& ListBlockFilterTypes(); diff --git a/src/bloom.cpp b/src/bloom.cpp index a061925089..bd6069b31f 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bloom.h b/src/bloom.h index 7d3aa878b0..68e76a0258 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -115,9 +115,6 @@ public: class CRollingBloomFilter { public: - // A random bloom filter calls GetRand() at creation time. - // Don't create global CRollingBloomFilter objects, as they may be - // constructed before the randomizer is properly initialized. CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); diff --git a/src/chain.cpp b/src/chain.cpp index 5520d8149a..c09113a866 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chain.h b/src/chain.h index 321bc95dbc..48bcb8bfdd 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -48,17 +48,15 @@ public: uint64_t nTimeFirst; //!< earliest time of block in file uint64_t nTimeLast; //!< latest time of block in file - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nBlocks)); - READWRITE(VARINT(nSize)); - READWRITE(VARINT(nUndoSize)); - READWRITE(VARINT(nHeightFirst)); - READWRITE(VARINT(nHeightLast)); - READWRITE(VARINT(nTimeFirst)); - READWRITE(VARINT(nTimeLast)); + SERIALIZE_METHODS(CBlockFileInfo, obj) + { + READWRITE(VARINT(obj.nBlocks)); + READWRITE(VARINT(obj.nSize)); + READWRITE(VARINT(obj.nUndoSize)); + READWRITE(VARINT(obj.nHeightFirst)); + READWRITE(VARINT(obj.nHeightLast)); + READWRITE(VARINT(obj.nTimeFirst)); + READWRITE(VARINT(obj.nTimeLast)); } void SetNull() { @@ -332,31 +330,25 @@ public: hashPrev = (pprev ? pprev->GetBlockHash() : uint256()); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CDiskBlockIndex, obj) + { int _nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); - - READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nStatus)); - READWRITE(VARINT(nTx)); - if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - if (nStatus & BLOCK_HAVE_DATA) - READWRITE(VARINT(nDataPos)); - if (nStatus & BLOCK_HAVE_UNDO) - READWRITE(VARINT(nUndoPos)); + if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); + + READWRITE(VARINT(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(obj.nStatus)); + READWRITE(VARINT(obj.nTx)); + if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED)); + if (obj.nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(obj.nDataPos)); + if (obj.nStatus & BLOCK_HAVE_UNDO) READWRITE(VARINT(obj.nUndoPos)); // block header - READWRITE(this->nVersion); - READWRITE(hashPrev); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); + READWRITE(obj.nVersion); + READWRITE(obj.hashPrev); + READWRITE(obj.hashMerkleRoot); + READWRITE(obj.nTime); + READWRITE(obj.nBits); + READWRITE(obj.nNonce); } uint256 GetBlockHash() const diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 7ff30ac1cd..31592b0f0a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -62,7 +62,7 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits class CMainParams : public CChainParams { public: CMainParams() { - strNetworkID = "main"; + strNetworkID = CBaseChainParams::MAIN; consensus.nSubsidyHalvingInterval = 210000; consensus.BIP16Exception = uint256S("0x00000000000002dc756eebf4f49723ed8d30cc28a5f108eb94b1ba88ac4f9c22"); consensus.BIP34Height = 227931; @@ -71,7 +71,7 @@ public: consensus.BIP66Height = 363725; // 00000000000000000379eaa19dce8c9b722d46ae6a57c2f1a988119488b50931 consensus.CSVHeight = 419328; // 000000000000000004a1b34462cb8aeebd5799177f7a29cf28f2d1961716b5b5 consensus.SegwitHeight = 481824; // 0000000000000000001c8018d9cb3b742ef25114f27563e3fc4a1902167f9893 - consensus.MinBIP9WarningHeight = consensus.SegwitHeight + consensus.nMinerConfirmationWindow; + consensus.MinBIP9WarningHeight = 483840; // segwit activation height + miner confirmation window consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -169,7 +169,7 @@ public: class CTestNetParams : public CChainParams { public: CTestNetParams() { - strNetworkID = "test"; + strNetworkID = CBaseChainParams::TESTNET; consensus.nSubsidyHalvingInterval = 210000; consensus.BIP16Exception = uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105"); consensus.BIP34Height = 21111; @@ -178,7 +178,7 @@ public: consensus.BIP66Height = 330776; // 000000002104c8c45e99a8853285a3b592602a3ccde2b832481da85e9e4ba182 consensus.CSVHeight = 770112; // 00000000025e930139bac5c6c31a403776da130831ab85be56578f3fa75369bb consensus.SegwitHeight = 834624; // 00000000002b980fcd729daaa248fd9316a5200e9b367f4ff2c42453e84201ca - consensus.MinBIP9WarningHeight = consensus.SegwitHeight + consensus.nMinerConfirmationWindow; + consensus.MinBIP9WarningHeight = 836640; // segwit activation height + miner confirmation window consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -254,7 +254,7 @@ public: class CRegTestParams : public CChainParams { public: explicit CRegTestParams(const ArgsManager& args) { - strNetworkID = "regtest"; + strNetworkID = CBaseChainParams::REGTEST; consensus.nSubsidyHalvingInterval = 150; consensus.BIP16Exception = uint256(); consensus.BIP34Height = 500; // BIP34 activated on regtest (Used in functional tests) diff --git a/src/chainparams.h b/src/chainparams.h index 8f1d27e03c..63398e587e 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,7 +75,7 @@ public: uint64_t AssumedChainStateSize() const { return m_assumed_chain_state_size; } /** Whether it is possible to mine blocks on demand (no retargeting) */ bool MineBlocksOnDemand() const { return consensus.fPowNoRetargeting; } - /** Return the BIP70 network string (main, test or regtest) */ + /** Return the network string */ std::string NetworkIDString() const { return strNetworkID; } /** Return the list of hostnames to look up for DNS seeds */ const std::vector<std::string>& DNSSeeds() const { return vSeeds; } @@ -120,7 +120,7 @@ std::unique_ptr<const CChainParams> CreateChainParams(const std::string& chain); const CChainParams &Params(); /** - * Sets the params returned by Params() to those for the given BIP70 chain name. + * Sets the params returned by Params() to those for the given chain name. * @throws std::runtime_error when the chain is not supported. */ void SelectParams(const std::string& chain); diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index 4bb66c8d8b..894b8553c4 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index f34646f7ac..3c139931ea 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,10 +15,12 @@ class CBaseChainParams { public: - /** BIP70 chain name strings (main, test or regtest) */ + ///@{ + /** Chain name strings */ static const std::string MAIN; static const std::string TESTNET; static const std::string REGTEST; + ///@} const std::string& DataDir() const { return strDataDir; } int RPCPort() const { return nRPCPort; } diff --git a/src/checkqueue.h b/src/checkqueue.h index 978e23a7c4..9dab5a09ac 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -90,8 +90,7 @@ private: nTotal--; bool fRet = fAllOk; // reset the status for new work later - if (fMaster) - fAllOk = true; + fAllOk = true; // return the current status return fRet; } diff --git a/src/coins.cpp b/src/coins.cpp index 6b85edd01a..b71362c6a0 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/coins.h b/src/coins.h index d8135e0d9a..e71c8a47bc 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -61,7 +61,7 @@ public: assert(!IsSpent()); uint32_t code = nHeight * 2 + fCoinBase; ::Serialize(s, VARINT(code)); - ::Serialize(s, CTxOutCompressor(REF(out))); + ::Serialize(s, Using<TxOutCompression>(out)); } template<typename Stream> @@ -70,7 +70,7 @@ public: ::Unserialize(s, VARINT(code)); nHeight = code >> 1; fCoinBase = code & 1; - ::Unserialize(s, CTxOutCompressor(out)); + ::Unserialize(s, Using<TxOutCompression>(out)); } bool IsSpent() const { diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index c254fe7cbf..27ef1a18df 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h new file mode 100644 index 0000000000..0877ad47d3 --- /dev/null +++ b/src/compat/cpuid.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_CPUID_H +#define BITCOIN_COMPAT_CPUID_H + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#define HAVE_GETCPUID + +#include <cpuid.h> + +// We can't use cpuid.h's __get_cpuid as it does not support subleafs. +void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + +#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#endif // BITCOIN_COMPAT_CPUID_H diff --git a/src/compat/stdin.cpp b/src/compat/stdin.cpp index 98d406cca8..0fc4e0fcf2 100644 --- a/src/compat/stdin.cpp +++ b/src/compat/stdin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compressor.cpp b/src/compressor.cpp index a7f45b5c1e..a70306d320 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compressor.h b/src/compressor.h index c1eda503c8..7bb60d311e 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -11,10 +11,6 @@ #include <serialize.h> #include <span.h> -class CKeyID; -class CPubKey; -class CScriptID; - bool CompressScript(const CScript& script, std::vector<unsigned char> &out); unsigned int GetSpecialScriptSize(unsigned int nSize); bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); @@ -33,9 +29,8 @@ uint64_t DecompressAmount(uint64_t nAmount); * Other scripts up to 121 bytes require 1 byte + script length. Above * that, scripts up to 16505 bytes require 2 bytes + script length. */ -class CScriptCompressor +struct ScriptCompression { -private: /** * make this static for now (there are only 6 special scripts defined) * this can potentially be extended together with a new nVersion for @@ -44,12 +39,8 @@ private: */ static const unsigned int nSpecialScripts = 6; - CScript &script; -public: - explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { } - template<typename Stream> - void Serialize(Stream &s) const { + void Ser(Stream &s, const CScript& script) { std::vector<unsigned char> compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); @@ -61,7 +52,7 @@ public: } template<typename Stream> - void Unserialize(Stream &s) { + void Unser(Stream &s, CScript& script) { unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { @@ -82,30 +73,24 @@ public: } }; -/** wrapper for CTxOut that provides a more compact serialization */ -class CTxOutCompressor +struct AmountCompression { -private: - CTxOut &txout; - -public: - explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { } - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (!ser_action.ForRead()) { - uint64_t nVal = CompressAmount(txout.nValue); - READWRITE(VARINT(nVal)); - } else { - uint64_t nVal = 0; - READWRITE(VARINT(nVal)); - txout.nValue = DecompressAmount(nVal); - } - CScriptCompressor cscript(REF(txout.scriptPubKey)); - READWRITE(cscript); + template<typename Stream, typename I> void Ser(Stream& s, I val) + { + s << VARINT(CompressAmount(val)); + } + template<typename Stream, typename I> void Unser(Stream& s, I& val) + { + uint64_t v; + s >> VARINT(v); + val = DecompressAmount(v); } }; +/** wrapper for CTxOut that provides a more compact serialization */ +struct TxOutCompression +{ + FORMATTER_METHODS(CTxOut, obj) { READWRITE(Using<AmountCompression>(obj.nValue), Using<ScriptCompression>(obj.scriptPubKey)); } +}; + #endif // BITCOIN_COMPRESSOR_H diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index f87612edef..843985e54c 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index f28f76bd34..4ae5a5b897 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/params.h b/src/consensus/params.h index e191fd6d26..61b1fbc2e5 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index 6793f871cf..bb8cd10c63 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,28 +7,28 @@ #include <primitives/transaction.h> #include <consensus/validation.h> -bool CheckTransaction(const CTransaction& tx, CValidationState& state) +bool CheckTransaction(const CTransaction& tx, TxValidationState& state) { // Basic checks that don't depend on any context if (tx.vin.empty()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-vin-empty"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vin-empty"); if (tx.vout.empty()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-vout-empty"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-empty"); // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-oversize"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-oversize"); // Check for negative or overflow output values (see CVE-2010-5139) CAmount nValueOut = 0; for (const auto& txout : tx.vout) { if (txout.nValue < 0) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-vout-negative"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-negative"); if (txout.nValue > MAX_MONEY) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-vout-toolarge"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-toolarge"); nValueOut += txout.nValue; if (!MoneyRange(nValueOut)) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-txouttotal-toolarge"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-txouttotal-toolarge"); } // Check for duplicate inputs (see CVE-2018-17144) @@ -39,19 +39,19 @@ bool CheckTransaction(const CTransaction& tx, CValidationState& state) std::set<COutPoint> vInOutPoints; for (const auto& txin : tx.vin) { if (!vInOutPoints.insert(txin.prevout).second) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-inputs-duplicate"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-inputs-duplicate"); } if (tx.IsCoinBase()) { if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-cb-length"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-cb-length"); } else { for (const auto& txin : tx.vin) if (txin.prevout.IsNull()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-prevout-null"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-prevout-null"); } return true; diff --git a/src/consensus/tx_check.h b/src/consensus/tx_check.h index 6f3f8fe969..21f842408a 100644 --- a/src/consensus/tx_check.h +++ b/src/consensus/tx_check.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,8 +13,8 @@ */ class CTransaction; -class CValidationState; +class TxValidationState; -bool CheckTransaction(const CTransaction& tx, CValidationState& state); +bool CheckTransaction(const CTransaction& tx, TxValidationState& state); #endif // BITCOIN_CONSENSUS_TX_CHECK_H diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index ceeddc3f6d..81245e3e11 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -156,11 +156,11 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i return nSigOps; } -bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee) +bool Consensus::CheckTxInputs(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee) { // are the actual inputs available? if (!inputs.HaveInputs(tx)) { - return state.Invalid(ValidationInvalidReason::TX_MISSING_INPUTS, false, "bad-txns-inputs-missingorspent", + return state.Invalid(TxValidationResult::TX_MISSING_INPUTS, "bad-txns-inputs-missingorspent", strprintf("%s: inputs missing/spent", __func__)); } @@ -172,27 +172,27 @@ bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, c // If prev is coinbase, check that it's matured if (coin.IsCoinBase() && nSpendHeight - coin.nHeight < COINBASE_MATURITY) { - return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, "bad-txns-premature-spend-of-coinbase", + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "bad-txns-premature-spend-of-coinbase", strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); } // Check for negative or overflow input values nValueIn += coin.out.nValue; if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-inputvalues-outofrange"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-inputvalues-outofrange"); } } const CAmount value_out = tx.GetValueOut(); if (nValueIn < value_out) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-in-belowout", + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-in-belowout", strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(value_out))); } // Tally transaction fees const CAmount txfee_aux = nValueIn - value_out; if (!MoneyRange(txfee_aux)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-fee-outofrange"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-fee-outofrange"); } txfee = txfee_aux; diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index 3519fc555d..ffcaf3cab1 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,7 @@ class CBlockIndex; class CCoinsViewCache; class CTransaction; -class CValidationState; +class TxValidationState; /** Transaction validation functions */ @@ -24,7 +24,7 @@ namespace Consensus { * @param[out] txfee Set to the transaction fee if successful. * Preconditions: tx.IsCoinBase() is false. */ -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee); +bool CheckTxInputs(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee); } // namespace Consensus /** Auxiliary functions for transaction validation (ideally should not be exposed) */ diff --git a/src/consensus/validation.h b/src/consensus/validation.h index 4920cdf881..8a3abb31f4 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,13 +12,12 @@ #include <primitives/transaction.h> #include <primitives/block.h> -/** A "reason" why something was invalid, suitable for determining whether the - * provider of the object should be banned/ignored/disconnected/etc. +/** A "reason" why a transaction was invalid, suitable for determining whether the + * provider of the transaction should be banned/ignored/disconnected/etc. */ -enum class ValidationInvalidReason { - // txn and blocks: - NONE, //!< not actually invalid - CONSENSUS, //!< invalid by consensus rules (excluding any below reasons) +enum class TxValidationResult { + TX_RESULT_UNSET, //!< initial value. Tx has not yet been rejected + TX_CONSENSUS, //!< invalid by consensus rules /** * Invalid by a change to consensus rules more recent than SegWit. * Currently unused as there are no such consensus rule changes, and any download @@ -26,18 +25,9 @@ enum class ValidationInvalidReason { * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork * is uninteresting. */ - RECENT_CONSENSUS_CHANGE, - // Only blocks (or headers): - CACHED_INVALID, //!< this object was cached as being invalid, but we don't know why - BLOCK_INVALID_HEADER, //!< invalid proof of work or time too old - BLOCK_MUTATED, //!< the block's data didn't match the data committed to by the PoW - BLOCK_MISSING_PREV, //!< We don't have the previous block the checked one is built on - BLOCK_INVALID_PREV, //!< A block this one builds on is invalid - BLOCK_TIME_FUTURE, //!< block timestamp was > 2 hours in the future (or our clock is bad) - BLOCK_CHECKPOINT, //!< the block failed to meet one of our checkpoints - // Only loose txn: + TX_RECENT_CONSENSUS_CHANGE, TX_NOT_STANDARD, //!< didn't meet our local policy rules - TX_MISSING_INPUTS, //!< a transaction was missing some of its inputs + TX_MISSING_INPUTS, //!< transaction was missing some of its inputs TX_PREMATURE_SPEND, //!< transaction spends a coinbase too early, or violates locktime/sequence locks /** * Transaction might be missing a witness, have a witness prior to SegWit @@ -48,82 +38,107 @@ enum class ValidationInvalidReason { /** * Tx already in mempool or conflicts with a tx in the chain * (if it conflicts with another tx in mempool, we use MEMPOOL_POLICY as it failed to reach the RBF threshold) - * TODO: Currently this is only used if the transaction already exists in the mempool or on chain, - * TODO: ATMP's fMissingInputs and a valid CValidationState being used to indicate missing inputs + * Currently this is only used if the transaction already exists in the mempool or on chain. */ TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits }; -inline bool IsTransactionReason(ValidationInvalidReason r) -{ - return r == ValidationInvalidReason::NONE || - r == ValidationInvalidReason::CONSENSUS || - r == ValidationInvalidReason::RECENT_CONSENSUS_CHANGE || - r == ValidationInvalidReason::TX_NOT_STANDARD || - r == ValidationInvalidReason::TX_PREMATURE_SPEND || - r == ValidationInvalidReason::TX_MISSING_INPUTS || - r == ValidationInvalidReason::TX_WITNESS_MUTATED || - r == ValidationInvalidReason::TX_CONFLICT || - r == ValidationInvalidReason::TX_MEMPOOL_POLICY; -} +/** A "reason" why a block was invalid, suitable for determining whether the + * provider of the block should be banned/ignored/disconnected/etc. + * These are much more granular than the rejection codes, which may be more + * useful for some other use-cases. + */ +enum class BlockValidationResult { + BLOCK_RESULT_UNSET, //!< initial value. Block has not yet been rejected + BLOCK_CONSENSUS, //!< invalid by consensus rules (excluding any below reasons) + /** + * Invalid by a change to consensus rules more recent than SegWit. + * Currently unused as there are no such consensus rule changes, and any download + * sources realistically need to support SegWit in order to provide useful data, + * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork + * is uninteresting. + */ + BLOCK_RECENT_CONSENSUS_CHANGE, + BLOCK_CACHED_INVALID, //!< this block was cached as being invalid and we didn't store the reason why + BLOCK_INVALID_HEADER, //!< invalid proof of work or time too old + BLOCK_MUTATED, //!< the block's data didn't match the data committed to by the PoW + BLOCK_MISSING_PREV, //!< We don't have the previous block the checked one is built on + BLOCK_INVALID_PREV, //!< A block this one builds on is invalid + BLOCK_TIME_FUTURE, //!< block timestamp was > 2 hours in the future (or our clock is bad) + BLOCK_CHECKPOINT, //!< the block failed to meet one of our checkpoints +}; -inline bool IsBlockReason(ValidationInvalidReason r) -{ - return r == ValidationInvalidReason::NONE || - r == ValidationInvalidReason::CONSENSUS || - r == ValidationInvalidReason::RECENT_CONSENSUS_CHANGE || - r == ValidationInvalidReason::CACHED_INVALID || - r == ValidationInvalidReason::BLOCK_INVALID_HEADER || - r == ValidationInvalidReason::BLOCK_MUTATED || - r == ValidationInvalidReason::BLOCK_MISSING_PREV || - r == ValidationInvalidReason::BLOCK_INVALID_PREV || - r == ValidationInvalidReason::BLOCK_TIME_FUTURE || - r == ValidationInvalidReason::BLOCK_CHECKPOINT; -} -/** Capture information about block/transaction validation */ -class CValidationState { + +/** Base class for capturing information about block/transaction validation. This is subclassed + * by TxValidationState and BlockValidationState for validation information on transactions + * and blocks respectively. */ +class ValidationState { private: enum mode_state { MODE_VALID, //!< everything ok MODE_INVALID, //!< network rule violation (DoS value may be set) MODE_ERROR, //!< run-time error - } mode; - ValidationInvalidReason m_reason; - std::string strRejectReason; - std::string strDebugMessage; -public: - CValidationState() : mode(MODE_VALID), m_reason(ValidationInvalidReason::NONE) {} - bool Invalid(ValidationInvalidReason reasonIn, bool ret = false, - const std::string &strRejectReasonIn="", - const std::string &strDebugMessageIn="") { - m_reason = reasonIn; - strRejectReason = strRejectReasonIn; - strDebugMessage = strDebugMessageIn; - if (mode == MODE_ERROR) - return ret; - mode = MODE_INVALID; - return ret; + } m_mode; + std::string m_reject_reason; + std::string m_debug_message; +protected: + void Invalid(const std::string &reject_reason="", + const std::string &debug_message="") + { + m_reject_reason = reject_reason; + m_debug_message = debug_message; + if (m_mode != MODE_ERROR) m_mode = MODE_INVALID; } - bool Error(const std::string& strRejectReasonIn) { - if (mode == MODE_VALID) - strRejectReason = strRejectReasonIn; - mode = MODE_ERROR; +public: + // ValidationState is abstract. Have a pure virtual destructor. + virtual ~ValidationState() = 0; + + ValidationState() : m_mode(MODE_VALID) {} + bool Error(const std::string& reject_reason) + { + if (m_mode == MODE_VALID) + m_reject_reason = reject_reason; + m_mode = MODE_ERROR; return false; } - bool IsValid() const { - return mode == MODE_VALID; - } - bool IsInvalid() const { - return mode == MODE_INVALID; + bool IsValid() const { return m_mode == MODE_VALID; } + bool IsInvalid() const { return m_mode == MODE_INVALID; } + bool IsError() const { return m_mode == MODE_ERROR; } + std::string GetRejectReason() const { return m_reject_reason; } + std::string GetDebugMessage() const { return m_debug_message; } +}; + +inline ValidationState::~ValidationState() {}; + +class TxValidationState : public ValidationState { +private: + TxValidationResult m_result = TxValidationResult::TX_RESULT_UNSET; +public: + bool Invalid(TxValidationResult result, + const std::string &reject_reason="", + const std::string &debug_message="") + { + m_result = result; + ValidationState::Invalid(reject_reason, debug_message); + return false; } - bool IsError() const { - return mode == MODE_ERROR; + TxValidationResult GetResult() const { return m_result; } +}; + +class BlockValidationState : public ValidationState { +private: + BlockValidationResult m_result = BlockValidationResult::BLOCK_RESULT_UNSET; +public: + bool Invalid(BlockValidationResult result, + const std::string &reject_reason="", + const std::string &debug_message="") { + m_result = result; + ValidationState::Invalid(reject_reason, debug_message); + return false; } - ValidationInvalidReason GetReason() const { return m_reason; } - std::string GetRejectReason() const { return strRejectReason; } - std::string GetDebugMessage() const { return strDebugMessage; } + BlockValidationResult GetResult() const { return m_result; } }; // These implement the weight = (stripped_size * 4) + witness_size formula, diff --git a/src/core_io.h b/src/core_io.h index 19fb7b29f6..80ec80cd50 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_read.cpp b/src/core_read.cpp index a3c9cf0159..9a65b02585 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_write.cpp b/src/core_write.cpp index 7ce2a49836..cb1fc214eb 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 6ed9088434..2afcbd1629 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/aes.h b/src/crypto/aes.h index e06c8de272..3a0011bee4 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 42a17f02ff..f3ff4268ee 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index 5a4674f4a8..69fbbe9fa5 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hkdf_sha256_32.cpp b/src/crypto/hkdf_sha256_32.cpp index 9cea5995ec..81f3c1349a 100644 --- a/src/crypto/hkdf_sha256_32.cpp +++ b/src/crypto/hkdf_sha256_32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ CHKDF_HMAC_SHA256_L32::CHKDF_HMAC_SHA256_L32(const unsigned char* ikm, size_t ikmlen, const std::string& salt) { - CHMAC_SHA256((const unsigned char*)salt.c_str(), salt.size()).Write(ikm, ikmlen).Finalize(m_prk); + CHMAC_SHA256((const unsigned char*)salt.data(), salt.size()).Write(ikm, ikmlen).Finalize(m_prk); } void CHKDF_HMAC_SHA256_L32::Expand32(const std::string& info, unsigned char hash[OUTPUT_SIZE]) diff --git a/src/crypto/ripemd160.cpp b/src/crypto/ripemd160.cpp index edee06cc34..29a4ad906f 100644 --- a/src/crypto/ripemd160.cpp +++ b/src/crypto/ripemd160.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha1.cpp b/src/crypto/sha1.cpp index 3dcdcb186e..1fb9bb2b72 100644 --- a/src/crypto/sha1.cpp +++ b/src/crypto/sha1.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 3257ee7f97..e35d526d35 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,9 +8,10 @@ #include <assert.h> #include <string.h> +#include <compat/cpuid.h> + #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #if defined(USE_ASM) -#include <cpuid.h> namespace sha256_sse4 { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); @@ -546,18 +547,7 @@ bool SelfTest() { return true; } - #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) -// We can't use cpuid.h's __get_cpuid as it does not support subleafs. -void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} - /** Check whether the OS has enabled AVX registers. */ bool AVXEnabled() { @@ -572,7 +562,7 @@ bool AVXEnabled() std::string SHA256AutoDetect() { std::string ret = "standard"; -#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) +#if defined(USE_ASM) && defined(HAVE_GETCPUID) bool have_sse4 = false; bool have_xsave = false; bool have_avx = false; @@ -589,7 +579,7 @@ std::string SHA256AutoDetect() (void)enabled_avx; uint32_t eax, ebx, ecx, edx; - cpuid(1, 0, eax, ebx, ecx, edx); + GetCPUID(1, 0, eax, ebx, ecx, edx); have_sse4 = (ecx >> 19) & 1; have_xsave = (ecx >> 27) & 1; have_avx = (ecx >> 28) & 1; @@ -597,7 +587,7 @@ std::string SHA256AutoDetect() enabled_avx = AVXEnabled(); } if (have_sse4) { - cpuid(7, 0, eax, ebx, ecx, edx); + GetCPUID(7, 0, eax, ebx, ecx, edx); have_avx2 = (ebx >> 5) & 1; have_shani = (ebx >> 29) & 1; } diff --git a/src/crypto/sha256_avx2.cpp b/src/crypto/sha256_avx2.cpp index 90a72516a4..624bdb42e4 100644 --- a/src/crypto/sha256_avx2.cpp +++ b/src/crypto/sha256_avx2.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifdef ENABLE_AVX2 #include <stdint.h> diff --git a/src/crypto/sha256_shani.cpp b/src/crypto/sha256_shani.cpp index 7ea0c34796..92f67710fb 100644 --- a/src/crypto/sha256_shani.cpp +++ b/src/crypto/sha256_shani.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/crypto/sha256_sse41.cpp b/src/crypto/sha256_sse41.cpp index fc79f46f7f..4eaf7d7b18 100644 --- a/src/crypto/sha256_sse41.cpp +++ b/src/crypto/sha256_sse41.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifdef ENABLE_SSE41 #include <stdint.h> diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 4e6aa363f7..85a7bbcb53 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index 4118ac1b18..21ca930c75 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ public: CSHA512& Write(const unsigned char* data, size_t len); void Finalize(unsigned char hash[OUTPUT_SIZE]); CSHA512& Reset(); + uint64_t Size() const { return bytes; } }; #endif // BITCOIN_CRYPTO_SHA512_H diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 4d0b094fa2..4ad5818cdc 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -6,50 +6,48 @@ #define BITCOIN_CUCKOOCACHE_H #include <array> -#include <algorithm> #include <atomic> -#include <cstring> #include <cmath> +#include <cstring> #include <memory> +#include <utility> #include <vector> -/** namespace CuckooCache provides high performance cache primitives +/** High-performance cache primitives. * * Summary: * - * 1) bit_packed_atomic_flags is bit-packed atomic flags for garbage collection + * 1. @ref bit_packed_atomic_flags is bit-packed atomic flags for garbage collection * - * 2) cache is a cache which is performant in memory usage and lookup speed. It - * is lockfree for erase operations. Elements are lazily erased on the next - * insert. + * 2. @ref cache is a cache which is performant in memory usage and lookup speed. It + * is lockfree for erase operations. Elements are lazily erased on the next insert. */ namespace CuckooCache { -/** bit_packed_atomic_flags implements a container for garbage collection flags +/** @ref bit_packed_atomic_flags implements a container for garbage collection flags * that is only thread unsafe on calls to setup. This class bit-packs collection * flags for memory efficiency. * - * All operations are std::memory_order_relaxed so external mechanisms must + * All operations are `std::memory_order_relaxed` so external mechanisms must * ensure that writes and reads are properly synchronized. * - * On setup(n), all bits up to n are marked as collected. + * On setup(n), all bits up to `n` are marked as collected. * * Under the hood, because it is an 8-bit type, it makes sense to use a multiple * of 8 for setup, but it will be safe if that is not the case as well. - * */ class bit_packed_atomic_flags { std::unique_ptr<std::atomic<uint8_t>[]> mem; public: - /** No default constructor as there must be some size */ + /** No default constructor, as there must be some size. */ bit_packed_atomic_flags() = delete; /** * bit_packed_atomic_flags constructor creates memory to sufficiently - * keep track of garbage collection information for size entries. + * keep track of garbage collection information for `size` entries. * * @param size the number of elements to allocate space for * @@ -68,7 +66,7 @@ public: }; /** setup marks all entries and ensures that bit_packed_atomic_flags can store - * at least size entries + * at least `b` entries. * * @param b the number of elements to allocate space for * @post bit_set, bit_unset, and bit_is_set function properly forall x. x < @@ -84,19 +82,18 @@ public: /** bit_set sets an entry as discardable. * - * @param s the index of the entry to bit_set. + * @param s the index of the entry to bit_set * @post immediately subsequent call (assuming proper external memory * ordering) to bit_is_set(s) == true. - * */ inline void bit_set(uint32_t s) { mem[s >> 3].fetch_or(1 << (s & 7), std::memory_order_relaxed); } - /** bit_unset marks an entry as something that should not be overwritten + /** bit_unset marks an entry as something that should not be overwritten. * - * @param s the index of the entry to bit_unset. + * @param s the index of the entry to bit_unset * @post immediately subsequent call (assuming proper external memory * ordering) to bit_is_set(s) == false. */ @@ -105,10 +102,10 @@ public: mem[s >> 3].fetch_and(~(1 << (s & 7)), std::memory_order_relaxed); } - /** bit_is_set queries the table for discardability at s + /** bit_is_set queries the table for discardability at `s`. * - * @param s the index of the entry to read. - * @returns if the bit at index s was set. + * @param s the index of the entry to read + * @returns true if the bit at index `s` was set, false otherwise * */ inline bool bit_is_set(uint32_t s) const { @@ -116,15 +113,15 @@ public: } }; -/** cache implements a cache with properties similar to a cuckoo-set +/** @ref cache implements a cache with properties similar to a cuckoo-set. * - * The cache is able to hold up to (~(uint32_t)0) - 1 elements. + * The cache is able to hold up to `(~(uint32_t)0) - 1` elements. * * Read Operations: - * - contains(*, false) + * - contains() for `erase=false` * * Read+Erase Operations: - * - contains(*, true) + * - contains() for `erase=true` * * Erase Operations: * - allow_erase() @@ -141,10 +138,10 @@ public: * * User Must Guarantee: * - * 1) Write Requires synchronized access (e.g., a lock) - * 2) Read Requires no concurrent Write, synchronized with the last insert. - * 3) Erase requires no concurrent Write, synchronized with last insert. - * 4) An Erase caller must release all memory before allowing a new Writer. + * 1. Write requires synchronized access (e.g. a lock) + * 2. Read requires no concurrent Write, synchronized with last insert. + * 3. Erase requires no concurrent Write, synchronized with last insert. + * 4. An Erase caller must release all memory before allowing a new Writer. * * * Note on function names: @@ -177,7 +174,7 @@ private: mutable std::vector<bool> epoch_flags; /** epoch_heuristic_counter is used to determine when an epoch might be aged - * & an expensive scan should be done. epoch_heuristic_counter is + * & an expensive scan should be done. epoch_heuristic_counter is * decremented on insert and reset to the new number of inserts which would * cause the epoch to reach epoch_size when it reaches zero. */ @@ -194,24 +191,25 @@ private: uint32_t epoch_size; /** depth_limit determines how many elements insert should try to replace. - * Should be set to log2(n)*/ + * Should be set to log2(n). + */ uint8_t depth_limit; /** hash_function is a const instance of the hash function. It cannot be * static or initialized at call time as it may have internal state (such as * a nonce). - * */ + */ const Hash hash_function; /** compute_hashes is convenience for not having to write out this * expression everywhere we use the hash values of an Element. * * We need to map the 32-bit input hash onto a hash bucket in a range [0, size) in a - * manner which preserves as much of the hash's uniformity as possible. Ideally + * manner which preserves as much of the hash's uniformity as possible. Ideally * this would be done by bitmasking but the size is usually not a power of two. * * The naive approach would be to use a mod -- which isn't perfectly uniform but so - * long as the hash is much larger than size it is not that bad. Unfortunately, + * long as the hash is much larger than size it is not that bad. Unfortunately, * mod/division is fairly slow on ordinary microprocessors (e.g. 90-ish cycles on * haswell, ARM doesn't even have an instruction for it.); when the divisor is a * constant the compiler will do clever tricks to turn it into a multiply+add+shift, @@ -223,10 +221,10 @@ private: * somewhat complicated and the result is still slower than other options: * * Instead we treat the 32-bit random number as a Q32 fixed-point number in the range - * [0,1) and simply multiply it by the size. Then we just shift the result down by - * 32-bits to get our bucket number. The result has non-uniformity the same as a + * [0, 1) and simply multiply it by the size. Then we just shift the result down by + * 32-bits to get our bucket number. The result has non-uniformity the same as a * mod, but it is much faster to compute. More about this technique can be found at - * http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + * http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ . * * The resulting non-uniformity is also more equally distributed which would be * advantageous for something like linear probing, though it shouldn't matter @@ -237,8 +235,8 @@ private: * 32*32->64 multiply, which means the operation is reasonably fast even on a * typical 32-bit processor. * - * @param e the element whose hashes will be returned - * @returns std::array<uint32_t, 8> of deterministic hashes derived from e + * @param e The element whose hashes will be returned + * @returns Deterministic hashes derived from `e` uniformly mapped onto the range [0, size) */ inline std::array<uint32_t, 8> compute_hashes(const Element& e) const { @@ -252,14 +250,14 @@ private: (uint32_t)(((uint64_t)hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}}; } - /* end - * @returns a constexpr index that can never be inserted to */ + /** invalid returns a special index that can never be inserted to + * @returns the special constexpr index that can never be inserted to */ constexpr uint32_t invalid() const { return ~(uint32_t)0; } - /** allow_erase marks the element at index n as discardable. Threadsafe + /** allow_erase marks the element at index `n` as discardable. Threadsafe * without any concurrent insert. * @param n the index to allow erasure of */ @@ -268,7 +266,7 @@ private: collection_flags.bit_set(n); } - /** please_keep marks the element at index n as an entry that should be kept. + /** please_keep marks the element at index `n` as an entry that should be kept. * Threadsafe without any concurrent insert. * @param n the index to prioritize keeping */ @@ -336,7 +334,7 @@ public: * * @param new_size the desired number of elements to store * @returns the maximum number of elements storable - **/ + */ uint32_t setup(uint32_t new_size) { // depth_limit must be at least one otherwise errors can occur. @@ -360,7 +358,7 @@ public: * negligible compared to the size of the elements. * * @param bytes the approximate number of bytes to use for this data - * structure. + * structure * @returns the maximum number of elements storable (see setup() * documentation for more detail) */ @@ -376,10 +374,12 @@ public: * It drops the last tried element if it runs out of depth before * encountering an open slot. * - * Thus + * Thus: * + * ``` * insert(x); * return contains(x, false); + * ``` * * is not guaranteed to return true. * @@ -387,7 +387,6 @@ public: * @post one of the following: All previously inserted elements and e are * now in the table, one previously inserted element is evicted from the * table, the entry attempted to be inserted is evicted. - * */ inline void insert(Element e) { @@ -416,9 +415,9 @@ public: /** Swap with the element at the location that was * not the last one looked at. Example: * - * 1) On first iteration, last_loc == invalid(), find returns last, so + * 1. On first iteration, last_loc == invalid(), find returns last, so * last_loc defaults to locs[0]. - * 2) On further iterations, where last_loc == locs[k], last_loc will + * 2. On further iterations, where last_loc == locs[k], last_loc will * go to locs[k+1 % 8], i.e., next of the 8 indices wrapping around * to 0 if needed. * @@ -439,17 +438,19 @@ public: } } - /* contains iterates through the hash locations for a given element + /** contains iterates through the hash locations for a given element * and checks to see if it is present. * * contains does not check garbage collected state (in other words, * garbage is only collected when the space is needed), so: * + * ``` * insert(x); * if (contains(x, true)) * return contains(x, false); * else * return true; + * ``` * * executed on a single thread will always return true! * @@ -458,7 +459,7 @@ public: * contains returns a bool set true if the element was found. * * @param e the element to check - * @param erase + * @param erase whether to attempt setting the garbage collect flag * * @post if erase is true and the element is found, then the garbage collect * flag is set diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 34896f7ab2..d7694108f5 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 061c9b6bca..116d7d8679 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 0edcb0286d..a5582e3b2c 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,8 @@ enum class WalletCreationStatus; namespace interfaces { class Chain; +class Handler; +class Wallet; } class DummyWalletInit : public WalletInitInterface { @@ -19,7 +21,7 @@ public: bool HasWalletSupport() const override {return false;} void AddWalletOptions() const override; bool ParameterInteraction() const override {return true;} - void Construct(InitInterfaces& interfaces) const override {LogPrintf("No wallet support compiled in!\n");} + void Construct(NodeContext& node) const override {LogPrintf("No wallet support compiled in!\n");} }; void DummyWalletInit::AddWalletOptions() const @@ -80,9 +82,13 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& throw std::logic_error("Wallet function called in non-wallet build."); } -namespace interfaces { +using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; +std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) +{ + throw std::logic_error("Wallet function called in non-wallet build."); +} -class Wallet; +namespace interfaces { std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { diff --git a/src/fs.cpp b/src/fs.cpp index 7b422b8d70..066c6c10d3 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <fs.h> #ifndef WIN32 @@ -107,10 +111,10 @@ std::string get_filesystem_error_message(const fs::filesystem_error& e) #else // Convert from Multi Byte to utf-16 std::string mb_string(e.what()); - int size = MultiByteToWideChar(CP_ACP, 0, mb_string.c_str(), mb_string.size(), nullptr, 0); + int size = MultiByteToWideChar(CP_ACP, 0, mb_string.data(), mb_string.size(), nullptr, 0); std::wstring utf16_string(size, L'\0'); - MultiByteToWideChar(CP_ACP, 0, mb_string.c_str(), mb_string.size(), &*utf16_string.begin(), size); + MultiByteToWideChar(CP_ACP, 0, mb_string.data(), mb_string.size(), &*utf16_string.begin(), size); // Convert from utf-16 to utf-8 return std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t>().to_bytes(utf16_string); #endif @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,6 @@ #include <ext/stdio_filebuf.h> #endif -#define BOOST_FILESYSTEM_NO_DEPRECATED #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 2c2f67b169..ff75789223 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,8 +15,13 @@ #include <util/translation.h> #include <walletinitinterface.h> +#include <algorithm> +#include <iterator> +#include <map> #include <memory> #include <stdio.h> +#include <set> +#include <string> #include <boost/algorithm/string.hpp> // boost::trim @@ -64,6 +69,9 @@ private: static std::string strRPCUserColonPass; /* Stored RPC timer interface (for unregistration) */ static std::unique_ptr<HTTPRPCTimerInterface> httpRPCTimerInterface; +/* RPC Auth Whitelist */ +static std::map<std::string, std::set<std::string>> g_rpc_whitelist; +static bool g_rpc_whitelist_default = false; static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id) { @@ -112,7 +120,7 @@ static bool multiUserAuthorized(std::string strUserPass) static const unsigned int KEY_SIZE = 32; unsigned char out[KEY_SIZE]; - CHMAC_SHA256(reinterpret_cast<const unsigned char*>(strSalt.c_str()), strSalt.size()).Write(reinterpret_cast<const unsigned char*>(strPass.c_str()), strPass.size()).Finalize(out); + CHMAC_SHA256(reinterpret_cast<const unsigned char*>(strSalt.data()), strSalt.size()).Write(reinterpret_cast<const unsigned char*>(strPass.data()), strPass.size()).Finalize(out); std::vector<unsigned char> hexvec(out, out+KEY_SIZE); std::string strHashFromPass = HexStr(hexvec); @@ -183,18 +191,45 @@ static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) jreq.URI = req->GetURI(); std::string strReply; + bool user_has_whitelist = g_rpc_whitelist.count(jreq.authUser); + if (!user_has_whitelist && g_rpc_whitelist_default) { + LogPrintf("RPC User %s not allowed to call any methods\n", jreq.authUser); + req->WriteReply(HTTP_FORBIDDEN); + return false; + // singleton request - if (valRequest.isObject()) { + } else if (valRequest.isObject()) { jreq.parse(valRequest); - + if (user_has_whitelist && !g_rpc_whitelist[jreq.authUser].count(jreq.strMethod)) { + LogPrintf("RPC User %s not allowed to call method %s\n", jreq.authUser, jreq.strMethod); + req->WriteReply(HTTP_FORBIDDEN); + return false; + } UniValue result = tableRPC.execute(jreq); // Send reply strReply = JSONRPCReply(result, NullUniValue, jreq.id); // array of requests - } else if (valRequest.isArray()) + } else if (valRequest.isArray()) { + if (user_has_whitelist) { + for (unsigned int reqIdx = 0; reqIdx < valRequest.size(); reqIdx++) { + if (!valRequest[reqIdx].isObject()) { + throw JSONRPCError(RPC_INVALID_REQUEST, "Invalid Request object"); + } else { + const UniValue& request = valRequest[reqIdx].get_obj(); + // Parse method + std::string strMethod = find_value(request, "method").get_str(); + if (!g_rpc_whitelist[jreq.authUser].count(strMethod)) { + LogPrintf("RPC User %s not allowed to call method %s\n", jreq.authUser, strMethod); + req->WriteReply(HTTP_FORBIDDEN); + return false; + } + } + } + } strReply = JSONRPCExecBatch(jreq, valRequest.get_array()); + } else throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); @@ -229,6 +264,27 @@ static bool InitRPCAuthentication() { LogPrintf("Using rpcauth authentication.\n"); } + + g_rpc_whitelist_default = gArgs.GetBoolArg("-rpcwhitelistdefault", gArgs.IsArgSet("-rpcwhitelist")); + for (const std::string& strRPCWhitelist : gArgs.GetArgs("-rpcwhitelist")) { + auto pos = strRPCWhitelist.find(':'); + std::string strUser = strRPCWhitelist.substr(0, pos); + bool intersect = g_rpc_whitelist.count(strUser); + std::set<std::string>& whitelist = g_rpc_whitelist[strUser]; + if (pos != std::string::npos) { + std::string strWhitelist = strRPCWhitelist.substr(pos + 1); + std::set<std::string> new_whitelist; + boost::split(new_whitelist, strWhitelist, boost::is_any_of(", ")); + if (intersect) { + std::set<std::string> tmp_whitelist; + std::set_intersection(new_whitelist.begin(), new_whitelist.end(), + whitelist.begin(), whitelist.end(), std::inserter(tmp_whitelist, tmp_whitelist.end())); + new_whitelist = std::move(tmp_whitelist); + } + whitelist = std::move(new_whitelist); + } + } + return true; } diff --git a/src/httprpc.h b/src/httprpc.h index 91c2ec0c9d..99e4d59b8a 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index d9c7113323..0e13b85806 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -172,7 +172,7 @@ static bool InitHTTPAllowList() rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; - LookupSubNet(strAllow.c_str(), subnet); + LookupSubNet(strAllow, subnet); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), @@ -324,7 +324,7 @@ static bool HTTPBindAddresses(struct evhttp* http) evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { CNetAddr addr; - if (i->first.empty() || (LookupHost(i->first.c_str(), addr, false) && addr.IsBindAny())) { + if (i->first.empty() || (LookupHost(i->first, addr, false) && addr.IsBindAny())) { LogPrintf("WARNING: the RPC server is not safe to expose to untrusted networks such as the public internet\n"); } boundSockets.push_back(bind_handle); diff --git a/src/httpserver.h b/src/httpserver.h index bc72fc8512..46820e6aee 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.cpp b/src/index/base.cpp index bcc8e2ce7c..dcb8e99fc1 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.h b/src/index/base.h index f95eeb8197..d0088d9c9a 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 62db38f894..5bbe6ad1df 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index da4d322669..e1a02edb96 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,6 +29,7 @@ #include <net_permissions.h> #include <net_processing.h> #include <netbase.h> +#include <node/context.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -51,12 +52,17 @@ #include <util/threadnames.h> #include <util/translation.h> #include <util/validation.h> +#include <util/asmap.h> #include <validation.h> +#include <hash.h> + + #include <validationinterface.h> #include <walletinitinterface.h> #include <stdint.h> #include <stdio.h> +#include <set> #ifndef WIN32 #include <attributes.h> @@ -84,9 +90,6 @@ static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; // Dump addresses to banlist.dat every 15 minutes (900s) static constexpr int DUMP_BANS_INTERVAL = 60 * 15; -std::unique_ptr<CConnman> g_connman; -std::unique_ptr<PeerLogicValidation> peerLogic; -std::unique_ptr<BanMan> g_banman; #ifdef WIN32 // Win32 LevelDB doesn't use filedescriptors, and the ones used for @@ -99,6 +102,8 @@ std::unique_ptr<BanMan> g_banman; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; + /** * The PID file facilities. */ @@ -154,7 +159,7 @@ static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; static boost::thread_group threadGroup; static CScheduler scheduler; -void Interrupt() +void Interrupt(NodeContext& node) { InterruptHTTPServer(); InterruptHTTPRPC(); @@ -162,18 +167,18 @@ void Interrupt() InterruptREST(); InterruptTorControl(); InterruptMapPort(); - if (g_connman) - g_connman->Interrupt(); + if (node.connman) + node.connman->Interrupt(); if (g_txindex) { g_txindex->Interrupt(); } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); } -void Shutdown(InitInterfaces& interfaces) +void Shutdown(NodeContext& node) { LogPrintf("%s: In progress...\n", __func__); - static CCriticalSection cs_Shutdown; + static RecursiveMutex cs_Shutdown; TRY_LOCK(cs_Shutdown, lockShutdown); if (!lockShutdown) return; @@ -189,17 +194,15 @@ void Shutdown(InitInterfaces& interfaces) StopREST(); StopRPC(); StopHTTPServer(); - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->flush(); } StopMapPort(); // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. - if (peerLogic) UnregisterValidationInterface(peerLogic.get()); - if (g_connman) g_connman->Stop(); - if (g_txindex) g_txindex->Stop(); - ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); + if (node.peer_logic) UnregisterValidationInterface(node.peer_logic.get()); + if (node.connman) node.connman->Stop(); StopTorControl(); @@ -210,11 +213,9 @@ void Shutdown(InitInterfaces& interfaces) // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. - peerLogic.reset(); - g_connman.reset(); - g_banman.reset(); - g_txindex.reset(); - DestroyAllBlockFilterIndexes(); + node.peer_logic.reset(); + node.connman.reset(); + node.banman.reset(); if (::mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(::mempool); @@ -247,6 +248,14 @@ void Shutdown(InitInterfaces& interfaces) // CValidationInterface callbacks, flush them... GetMainSignals().FlushBackgroundCallbacks(); + // Stop and delete all indexes only after flushing background callbacks. + if (g_txindex) { + g_txindex->Stop(); + g_txindex.reset(); + } + ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); + DestroyAllBlockFilterIndexes(); + // Any future callbacks will be dropped. This should absolutely be safe - if // missing a callback results in an unrecoverable situation, unclean shutdown // would too. The only reason to do the above flushes is to let the wallet catch @@ -261,7 +270,7 @@ void Shutdown(InitInterfaces& interfaces) } pblocktree.reset(); } - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->stop(); } @@ -280,12 +289,12 @@ void Shutdown(InitInterfaces& interfaces) } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } - interfaces.chain_clients.clear(); + node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - GetMainSignals().UnregisterWithMempoolSignals(mempool); globalVerifyHandle.reset(); ECC_Stop(); + if (node.mempool) node.mempool = nullptr; LogPrintf("%s: done\n", __func__); } @@ -374,7 +383,7 @@ void SetupServerArgs() gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -427,7 +436,8 @@ void SetupServerArgs() gArgs.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); + gArgs.AddArg("-asmap=<file>", "Specify asn mapping used for bucketing of the peers. Path should be relative to the -datadir path.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -482,7 +492,7 @@ void SetupServerArgs() "(0-4, default: %u)", DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-checkpoints", strprintf("Disable expensive verification for known chain history (default: %u)", DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + gArgs.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block 295000 (default: %u)", DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-deprecatedrpc=<method>", "Allows deprecated RPC method(s) to be used", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -499,7 +509,7 @@ void SetupServerArgs() gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-mocktime=<n>", "Replace actual time with <n> seconds since epoch (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + gArgs.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -518,7 +528,7 @@ void SetupServerArgs() gArgs.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted inbound peers with default permissions. This will relay transactions even if the transactions were already in the mempool or violate local relay policy. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. The will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. This will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-blockmaxweight=<n>", strprintf("Set maximum BIP141 block weight (default: %d)", DEFAULT_BLOCK_MAX_WEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); @@ -527,15 +537,17 @@ void SetupServerArgs() gArgs.AddArg("-rest", strprintf("Accept public REST requests (default: %u)", DEFAULT_REST_ENABLE), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); gArgs.AddArg("-rpcallowip=<ip>", "Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcauth=<userpw>", "Username and HMAC-SHA-256 hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::RPC); + gArgs.AddArg("-rpcauth=<userpw>", "Username and HMAC-SHA-256 hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); + gArgs.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY | ArgsManager::SENSITIVE, OptionsCategory::RPC); gArgs.AddArg("-rpccookiefile=<loc>", "Location of the auth cookie. Relative paths will be prefixed by a net-specific datadir location. (default: data dir)", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); + gArgs.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); gArgs.AddArg("-rpcport=<port>", strprintf("Listen for JSON-RPC connections on <port> (default: %u, testnet: %u, regtest: %u)", defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort(), regtestBaseParams->RPCPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::RPC); gArgs.AddArg("-rpcserialversion", strprintf("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)", DEFAULT_RPC_SERIALIZE_VERSION), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); gArgs.AddArg("-rpcservertimeout=<n>", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); gArgs.AddArg("-rpcthreads=<n>", strprintf("Set the number of threads to service RPC calls (default: %d)", DEFAULT_HTTP_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); + gArgs.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); + gArgs.AddArg("-rpcwhitelist=<whitelist>", "Set a whitelist to filter incoming RPC calls for a specific user. The field <whitelist> comes in the format: <USERNAME>:<rpc 1>,<rpc 2>,...,<rpc n>. If multiple whitelists are set for a given user, they are set-intersected. See -rpcwhitelistdefault documentation for information on default whitelist behavior.", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); + gArgs.AddArg("-rpcwhitelistdefault", "Sets default behavior for rpc whitelisting. Unless rpcwhitelistdefault is set to 0, if any -rpcwhitelist is set, the rpc server acts as if all rpc users are subject to empty-unless-otherwise-specified whitelists. If rpcwhitelistdefault is set to 1 and no -rpcwhitelist is set, rpc server acts as if all rpc users are subject to empty whitelists.", ArgsManager::ALLOW_BOOL, OptionsCategory::RPC); gArgs.AddArg("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); gArgs.AddArg("-server", "Accept command line and JSON-RPC commands", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); @@ -565,9 +577,7 @@ std::string LicenseInfo() "\n" + "\n" + _("This is experimental software.").translated + "\n" + - strprintf(_("Distributed under the MIT software license, see the accompanying file %s or %s").translated, "COPYING", "<https://opensource.org/licenses/MIT>") + "\n" + - "\n" + - strprintf(_("This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit %s and cryptographic software written by Eric Young and UPnP software written by Thomas Bernard.").translated, "<https://www.openssl.org>") + + strprintf(_("Distributed under the MIT software license, see the accompanying file %s or %s").translated, "COPYING", "<https://opensource.org/licenses/MIT>") + "\n"; } @@ -686,20 +696,6 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) LoadGenesisBlock(chainparams); } - // hardcoded $DATADIR/bootstrap.dat - fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (fs::exists(pathBootstrap)) { - FILE *file = fsbridge::fopen(pathBootstrap, "rb"); - if (file) { - fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; - LogPrintf("Importing bootstrap.dat...\n"); - LoadExternalBlockFile(chainparams, file); - RenameOver(pathBootstrap, pathBootstrapOld); - } else { - LogPrintf("Warning: Could not open bootstrap file %s\n", pathBootstrap.string()); - } - } - // -loadblock= for (const fs::path& path : vImportFiles) { FILE *file = fsbridge::fopen(path, "rb"); @@ -712,7 +708,7 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } // scan for better chains in the block chain database, that are not yet connected in the active best chain - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, chainparams)) { LogPrintf("Failed to connect best block (%s)\n", FormatStateMessage(state)); StartShutdown(); @@ -864,7 +860,7 @@ int nUserMaxConnections; int nFD; ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); int64_t peer_connect_timeout; -std::vector<BlockFilterType> g_enabled_filter_types; +std::set<BlockFilterType> g_enabled_filter_types; } // namespace @@ -892,8 +888,8 @@ bool AppInitBasicSetup() _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); #endif #ifdef WIN32 - // Enable Data Execution Prevention (DEP) - SetProcessDEPPolicy(PROCESS_DEP_ENABLE); + // Enable heap terminate-on-corruption + HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif if (!SetupNetworking()) @@ -943,7 +939,7 @@ bool AppInitParameterInteraction() } if (!fs::is_directory(GetBlocksDir())) { - return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.").translated, gArgs.GetArg("-blocksdir", "").c_str())); + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.").translated, gArgs.GetArg("-blocksdir", ""))); } // parse and validate enabled filter types @@ -952,13 +948,12 @@ bool AppInitParameterInteraction() g_enabled_filter_types = AllBlockFilterTypes(); } else if (blockfilterindex_value != "0") { const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex"); - g_enabled_filter_types.reserve(names.size()); for (const auto& name : names) { BlockFilterType filter_type; if (!BlockFilterTypeByName(name, filter_type)) { return InitError(strprintf(_("Unknown -blockfilterindex value %s.").translated, name)); } - g_enabled_filter_types.push_back(filter_type); + g_enabled_filter_types.insert(filter_type); } } @@ -1063,15 +1058,6 @@ bool AppInitParameterInteraction() incrementalRelayFee = CFeeRate(n); } - // -par=0 means autodetect, but nScriptCheckThreads==0 means no concurrency - nScriptCheckThreads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); - if (nScriptCheckThreads <= 0) - nScriptCheckThreads += GetNumCores(); - if (nScriptCheckThreads <= 1) - nScriptCheckThreads = 0; - else if (nScriptCheckThreads > MAX_SCRIPTCHECK_THREADS) - nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS; - // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg = gArgs.GetArg("-prune", 0); if (nPruneArg < 0) { @@ -1207,7 +1193,7 @@ bool AppInitLockDataDirectory() return true; } -bool AppInitMain(InitInterfaces& interfaces) +bool AppInitMain(NodeContext& node) { const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization @@ -1244,6 +1230,9 @@ bool AppInitMain(InitInterfaces& interfaces) LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); } + // Log the config arguments to debug.log + gArgs.LogArgs(); + LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. @@ -1258,33 +1247,52 @@ bool AppInitMain(InitInterfaces& interfaces) InitSignatureCache(); InitScriptExecutionCache(); - LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); - if (nScriptCheckThreads) { - for (int i=0; i<nScriptCheckThreads-1; i++) + int script_threads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + if (script_threads <= 0) { + // -par=0 means autodetect (number of cores - 1 script threads) + // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) + script_threads += GetNumCores(); + } + + // Subtract 1 because the main thread counts towards the par threads + script_threads = std::max(script_threads - 1, 0); + + // Number of script-checking threads <= MAX_SCRIPTCHECK_THREADS + script_threads = std::min(script_threads, MAX_SCRIPTCHECK_THREADS); + + LogPrintf("Script verification uses %d additional threads\n", script_threads); + if (script_threads >= 1) { + g_parallel_script_checks = true; + for (int i = 0; i < script_threads; ++i) { threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); + } } // Start the lightweight task scheduler thread CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + // Gather some entropy once per minute. + scheduler.scheduleEvery([]{ + RandAddPeriodic(); + }, 60000); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - GetMainSignals().RegisterWithMempoolSignals(mempool); // Create client interfaces for wallets that are supposed to be loaded // according to -wallet and -disablewallet options. This only constructs // the interfaces, it doesn't load wallet data. Wallets actually get loaded // when load() and start() interface methods are called below. - g_wallet_init_interface.Construct(interfaces); + g_wallet_init_interface.Construct(node); /* Register RPC commands regardless of -server setting so they will be * available in the GUI RPC console even if external calls are disabled. */ RegisterAllCoreRPCCommands(tableRPC); - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->registerRpcs(); } - g_rpc_interfaces = &interfaces; + g_rpc_node = &node; #if ENABLE_ZMQ RegisterZMQRPCCommands(tableRPC); #endif @@ -1302,7 +1310,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ********************************************************* Step 5: verify wallet database integrity - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { if (!client->verify()) { return false; } @@ -1314,13 +1322,13 @@ bool AppInitMain(InitInterfaces& interfaces) // is not yet setup and may end up being set up twice if we // need to reindex later. - assert(!g_banman); - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); - assert(!g_connman); - g_connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); + assert(!node.banman); + node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + assert(!node.connman); + node.connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); - peerLogic.reset(new PeerLogicValidation(g_connman.get(), g_banman.get(), scheduler)); - RegisterValidationInterface(peerLogic.get()); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), scheduler)); + RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; @@ -1360,7 +1368,7 @@ bool AppInitMain(InitInterfaces& interfaces) SetReachable(NET_ONION, false); if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; - if (!Lookup(proxyArg.c_str(), proxyAddr, 9050, fNameLookup)) { + if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'").translated, proxyArg)); } @@ -1384,7 +1392,7 @@ bool AppInitMain(InitInterfaces& interfaces) SetReachable(NET_ONION, false); } else { CService onionProxy; - if (!Lookup(onionArg.c_str(), onionProxy, 9050, fNameLookup)) { + if (!Lookup(onionArg, onionProxy, 9050, fNameLookup)) { return InitError(strprintf(_("Invalid -onion address or hostname: '%s'").translated, onionArg)); } proxyType addrOnion = proxyType(onionProxy, proxyRandomize); @@ -1402,7 +1410,7 @@ bool AppInitMain(InitInterfaces& interfaces) for (const std::string& strAddr : gArgs.GetArgs("-externalip")) { CService addrLocal; - if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) + if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); else return InitError(ResolveErrMsg("externalip", strAddr)); @@ -1642,6 +1650,11 @@ bool AppInitMain(InitInterfaces& interfaces) return false; } + // Now that the chain state is loaded, make mempool generally available in the node context. For example the + // connection manager, wallet, or RPC threads, which are all started after this, may use it from the node context. + assert(!node.mempool); + node.mempool = &::mempool; + fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); // Allowed to fail as this file IS missing on first startup. @@ -1661,7 +1674,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ********************************************************* Step 9: load wallet - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { if (!client->load()) { return false; } @@ -1765,8 +1778,8 @@ bool AppInitMain(InitInterfaces& interfaces) connOptions.nMaxFeeler = 1; connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; - connOptions.m_banman = g_banman.get(); - connOptions.m_msgproc = peerLogic.get(); + connOptions.m_banman = node.banman.get(); + connOptions.m_msgproc = node.peer_logic.get(); connOptions.nSendBufferMaxSize = 1000*gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000*gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = gArgs.GetArgs("-addnode"); @@ -1777,7 +1790,7 @@ bool AppInitMain(InitInterfaces& interfaces) for (const std::string& strBind : gArgs.GetArgs("-bind")) { CService addrBind; - if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) { + if (!Lookup(strBind, addrBind, GetListenPort(), false)) { return InitError(ResolveErrMsg("bind", strBind)); } connOptions.vBinds.push_back(addrBind); @@ -1806,21 +1819,41 @@ bool AppInitMain(InitInterfaces& interfaces) connOptions.m_specified_outgoing = connect; } } - if (!g_connman->Start(scheduler, connOptions)) { + if (!node.connman->Start(scheduler, connOptions)) { return false; } + // Read asmap file if configured + if (gArgs.IsArgSet("-asmap")) { + std::string asmap_file = gArgs.GetArg("-asmap", ""); + if (asmap_file.empty()) { + asmap_file = DEFAULT_ASMAP_FILENAME; + } + const fs::path asmap_path = GetDataDir() / asmap_file; + std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not find or parse specified asmap: '%s'").translated, asmap_path)); + return false; + } + node.connman->SetAsmap(asmap); + const uint256 asmap_version = SerializeHash(asmap); + LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing.\n"); + } + // ********************************************************* Step 13: finished SetRPCWarmupFinished(); uiInterface.InitMessage(_("Done loading").translated); - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->start(scheduler); } - scheduler.scheduleEvery([]{ - g_banman->DumpBanlist(); + BanMan* banman = node.banman.get(); + scheduler.scheduleEvery([banman]{ + banman->DumpBanlist(); }, DUMP_BANS_INTERVAL * 1000); return true; diff --git a/src/init.h b/src/init.h index 1c59ca069e..f74ae5a47a 100644 --- a/src/init.h +++ b/src/init.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,26 +10,14 @@ #include <string> #include <util/system.h> -namespace interfaces { -class Chain; -class ChainClient; -} // namespace interfaces - -//! Pointers to interfaces used during init and destroyed on shutdown. -struct InitInterfaces -{ - std::unique_ptr<interfaces::Chain> chain; - std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; -}; - -namespace boost -{ +struct NodeContext; +namespace boost { class thread_group; } // namespace boost /** Interrupt threads */ -void Interrupt(); -void Shutdown(InitInterfaces& interfaces); +void Interrupt(NodeContext& node); +void Shutdown(NodeContext& node); //!Initialize the logging infrastructure void InitLogging(); //!Parameter interaction: change current parameters depending on various rules @@ -63,7 +51,7 @@ bool AppInitLockDataDirectory(); * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(InitInterfaces& interfaces); +bool AppInitMain(NodeContext& node); /** * Setup the arguments for gArgs diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index b2c20573fb..643bb58d56 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,7 @@ #include <net.h> #include <net_processing.h> #include <node/coin.h> +#include <node/context.h> #include <node/transaction.h> #include <policy/fees.h> #include <policy/policy.h> @@ -37,7 +38,7 @@ namespace interfaces { namespace { -class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> +class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> { Optional<int> getHeight() override { @@ -57,12 +58,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } return nullopt; } - int getBlockDepth(const uint256& hash) override - { - const Optional<int> tip_height = getHeight(); - const Optional<int> height = getBlockHeight(hash); - return tip_height && height ? *tip_height - *height + 1 : 0; - } uint256 getBlockHash(int height) override { LockAssertion lock(::cs_main); @@ -181,11 +176,11 @@ public: const CBlockIndex* index, const std::vector<CTransactionRef>& tx_conflicted) override { - m_notifications->BlockConnected(*block, tx_conflicted); + m_notifications->BlockConnected(*block, tx_conflicted, index->nHeight); } - void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->BlockDisconnected(*block); + m_notifications->BlockDisconnected(*block, index->nHeight); } void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override { @@ -238,6 +233,7 @@ public: class ChainImpl : public Chain { public: + explicit ChainImpl(NodeContext& node) : m_node(node) {} std::unique_ptr<Chain::Lock> lock(bool try_lock) override { auto result = MakeUnique<LockImpl>(::cs_main, "cs_main", __FILE__, __LINE__, try_lock); @@ -267,7 +263,7 @@ public: } return true; } - void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(coins); } + void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); @@ -286,7 +282,7 @@ public: } bool broadcastTransaction(const CTransactionRef& tx, std::string& err_string, const CAmount& max_tx_fee, bool relay) override { - const TransactionError err = BroadcastTransaction(tx, err_string, max_tx_fee, relay, /*wait_callback*/ false); + const TransactionError err = BroadcastTransaction(m_node, tx, err_string, max_tx_fee, relay, /*wait_callback*/ false); // Chain clients only care about failures to accept the tx to the mempool. Disregard non-mempool related failures. // Note: this will need to be updated if BroadcastTransactions() is updated to return other non-mempool failures // that Chain clients do not need to know about. @@ -342,7 +338,6 @@ public: void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } void initWarning(const std::string& message) override { InitWarning(message); } void initError(const std::string& message) override { InitError(message); } - void loadWallet(std::unique_ptr<Wallet> wallet) override { ::uiInterface.LoadWallet(wallet); } void showProgress(const std::string& title, int progress, bool resume_possible) override { ::uiInterface.ShowProgress(title, progress, resume_possible); @@ -351,13 +346,11 @@ public: { return MakeUnique<NotificationsHandlerImpl>(*this, notifications); } - void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) override + void waitForNotificationsIfTipChanged(const uint256& old_tip) override { if (!old_tip.IsNull()) { LOCK(::cs_main); if (old_tip == ::ChainActive().Tip()->GetBlockHash()) return; - CBlockIndex* block = LookupBlockIndex(old_tip); - if (block && block->GetAncestor(::ChainActive().Height()) == ::ChainActive().Tip()) return; } SyncWithValidationInterfaceQueue(); } @@ -378,9 +371,10 @@ public: notifications.TransactionAddedToMempool(entry.GetSharedTx()); } } + NodeContext& m_node; }; } // namespace -std::unique_ptr<Chain> MakeChain() { return MakeUnique<ChainImpl>(); } +std::unique_ptr<Chain> MakeChain(NodeContext& node) { return MakeUnique<ChainImpl>(node); } } // namespace interfaces diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 73a78e21fb..7304f82749 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,12 +18,12 @@ class CBlock; class CFeeRate; class CRPCCommand; class CScheduler; -class CValidationState; class Coin; class uint256; enum class RBFTransactionState; struct CBlockLocator; struct FeeCalculation; +struct NodeContext; namespace interfaces { @@ -43,7 +43,7 @@ class Wallet; //! asynchronously //! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). //! -//! * The initMessages() and loadWallet() methods which the wallet uses to send +//! * The initMessage() and showProgress() methods which the wallet uses to send //! notifications to the GUI should go away when GUI and wallet can directly //! communicate with each other without going through the node //! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). @@ -76,10 +76,6 @@ public: //! included in the current chain. virtual Optional<int> getBlockHeight(const uint256& hash) = 0; - //! Get block depth. Returns 1 for chain tip, 2 for preceding block, and - //! so on. Returns 0 for a block not included in the current chain. - virtual int getBlockDepth(const uint256& hash) = 0; - //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; @@ -213,9 +209,6 @@ public: //! Send init error. virtual void initError(const std::string& message) = 0; - //! Send wallet load notification to the GUI. - virtual void loadWallet(std::unique_ptr<Wallet> wallet) = 0; - //! Send progress indicator. virtual void showProgress(const std::string& title, int progress, bool resume_possible) = 0; @@ -226,8 +219,8 @@ public: virtual ~Notifications() {} virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} virtual void TransactionRemovedFromMempool(const CTransactionRef& ptx) {} - virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted) {} - virtual void BlockDisconnected(const CBlock& block) {} + virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted, int height) {} + virtual void BlockDisconnected(const CBlock& block, int height) {} virtual void UpdatedBlockTip() {} virtual void ChainStateFlushed(const CBlockLocator& locator) {} }; @@ -236,9 +229,8 @@ public: virtual std::unique_ptr<Handler> handleNotifications(Notifications& notifications) = 0; //! Wait for pending notifications to be processed unless block hash points to the current - //! chain tip, or to a possible descendant of the current chain tip that isn't currently - //! connected. - virtual void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) = 0; + //! chain tip. + virtual void waitForNotificationsIfTipChanged(const uint256& old_tip) = 0; //! Register handler for RPC. Command is not copied, so reference //! needs to remain valid until Handler is disconnected. @@ -291,7 +283,7 @@ public: }; //! Return implementation of Chain interface. -std::unique_ptr<Chain> MakeChain(); +std::unique_ptr<Chain> MakeChain(NodeContext& node); //! Return implementation of ChainClient interface for a wallet client. This //! function will be undefined in builds where ENABLE_WALLET is false. diff --git a/src/interfaces/handler.cpp b/src/interfaces/handler.cpp index 92601fc4e9..95035c1b54 100644 --- a/src/interfaces/handler.cpp +++ b/src/interfaces/handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,6 +22,15 @@ public: boost::signals2::scoped_connection m_connection; }; +class CleanupHandler : public Handler +{ +public: + explicit CleanupHandler(std::function<void()> cleanup) : m_cleanup(std::move(cleanup)) {} + ~CleanupHandler() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } + void disconnect() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } + std::function<void()> m_cleanup; +}; + } // namespace std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection) @@ -29,4 +38,9 @@ std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection) return MakeUnique<HandlerImpl>(std::move(connection)); } +std::unique_ptr<Handler> MakeHandler(std::function<void()> cleanup) +{ + return MakeUnique<CleanupHandler>(std::move(cleanup)); +} + } // namespace interfaces diff --git a/src/interfaces/handler.h b/src/interfaces/handler.h index c4c674cac5..fbac3c6b71 100644 --- a/src/interfaces/handler.h +++ b/src/interfaces/handler.h @@ -1,10 +1,11 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_INTERFACES_HANDLER_H #define BITCOIN_INTERFACES_HANDLER_H +#include <functional> #include <memory> namespace boost { @@ -30,6 +31,9 @@ public: //! Return handler wrapping a boost signal connection. std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection); +//! Return handler wrapping a cleanup function. +std::unique_ptr<Handler> MakeHandler(std::function<void()> cleanup); + } // namespace interfaces #endif // BITCOIN_INTERFACES_HANDLER_H diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 227ac9f7b9..8a64a9d26a 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,6 +16,7 @@ #include <net_processing.h> #include <netaddress.h> #include <netbase.h> +#include <node/context.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/settings.h> @@ -42,17 +43,15 @@ std::vector<fs::path> ListWalletDir(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings); WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); +std::unique_ptr<interfaces::Handler> HandleLoadWallet(interfaces::Node::LoadWalletFn load_wallet); namespace interfaces { -class Wallet; - namespace { class NodeImpl : public Node { public: - NodeImpl() { m_interfaces.chain = MakeChain(); } void initError(const std::string& message) override { InitError(message); } bool parseParameters(int argc, const char* const argv[], std::string& error) override { @@ -68,18 +67,22 @@ public: std::string getNetwork() override { return Params().NetworkIDString(); } void initLogging() override { InitLogging(); } void initParameterInteraction() override { InitParameterInteraction(); } - std::string getWarnings(const std::string& type) override { return GetWarnings(type); } + std::string getWarnings() override { return GetWarnings(true); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { return AppInitBasicSetup() && AppInitParameterInteraction() && AppInitSanityChecks() && AppInitLockDataDirectory(); } - bool appInitMain() override { return AppInitMain(m_interfaces); } + bool appInitMain() override + { + m_context.chain = MakeChain(m_context); + return AppInitMain(m_context); + } void appShutdown() override { - Interrupt(); - Shutdown(m_interfaces); + Interrupt(m_context); + Shutdown(m_context); } void startShutdown() override { StartShutdown(); } bool shutdownRequested() override { return ShutdownRequested(); } @@ -96,15 +99,15 @@ public: bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { - return g_connman ? g_connman->GetNodeCount(flags) : 0; + return m_context.connman ? m_context.connman->GetNodeCount(flags) : 0; } bool getNodesStats(NodesStats& stats) override { stats.clear(); - if (g_connman) { + if (m_context.connman) { std::vector<CNodeStats> stats_temp; - g_connman->GetNodeStats(stats_temp); + m_context.connman->GetNodeStats(stats_temp); stats.reserve(stats_temp.size()); for (auto& node_stats_temp : stats_temp) { @@ -125,46 +128,46 @@ public: } bool getBanned(banmap_t& banmap) override { - if (g_banman) { - g_banman->GetBanned(banmap); + if (m_context.banman) { + m_context.banman->GetBanned(banmap); return true; } return false; } bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) override { - if (g_banman) { - g_banman->Ban(net_addr, reason, ban_time_offset); + if (m_context.banman) { + m_context.banman->Ban(net_addr, reason, ban_time_offset); return true; } return false; } bool unban(const CSubNet& ip) override { - if (g_banman) { - g_banman->Unban(ip); + if (m_context.banman) { + m_context.banman->Unban(ip); return true; } return false; } bool disconnect(const CNetAddr& net_addr) override { - if (g_connman) { - return g_connman->DisconnectNode(net_addr); + if (m_context.connman) { + return m_context.connman->DisconnectNode(net_addr); } return false; } bool disconnect(NodeId id) override { - if (g_connman) { - return g_connman->DisconnectNode(id); + if (m_context.connman) { + return m_context.connman->DisconnectNode(id); } return false; } - int64_t getTotalBytesRecv() override { return g_connman ? g_connman->GetTotalBytesRecv() : 0; } - int64_t getTotalBytesSent() override { return g_connman ? g_connman->GetTotalBytesSent() : 0; } - size_t getMempoolSize() override { return ::mempool.size(); } - size_t getMempoolDynamicUsage() override { return ::mempool.DynamicMemoryUsage(); } + int64_t getTotalBytesRecv() override { return m_context.connman ? m_context.connman->GetTotalBytesRecv() : 0; } + int64_t getTotalBytesSent() override { return m_context.connman ? m_context.connman->GetTotalBytesSent() : 0; } + size_t getMempoolSize() override { return m_context.mempool ? m_context.mempool->size() : 0; } + size_t getMempoolDynamicUsage() override { return m_context.mempool ? m_context.mempool->DynamicMemoryUsage() : 0; } bool getHeaderTip(int& height, int64_t& block_time) override { LOCK(::cs_main); @@ -202,11 +205,11 @@ public: bool getImporting() override { return ::fImporting; } void setNetworkActive(bool active) override { - if (g_connman) { - g_connman->SetNetworkActive(active); + if (m_context.connman) { + m_context.connman->SetNetworkActive(active); } } - bool getNetworkActive() override { return g_connman && g_connman->GetNetworkActive(); } + bool getNetworkActive() override { return m_context.connman && m_context.connman->GetNetworkActive(); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) override { FeeCalculation fee_calc; @@ -255,12 +258,12 @@ public: } std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::vector<std::string>& warnings) override { - return MakeWallet(LoadWallet(*m_interfaces.chain, name, error, warnings)); + return MakeWallet(LoadWallet(*m_context.chain, name, error, warnings)); } WalletCreationStatus createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::unique_ptr<Wallet>& result) override { std::shared_ptr<CWallet> wallet; - WalletCreationStatus status = CreateWallet(*m_interfaces.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); + WalletCreationStatus status = CreateWallet(*m_context.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); result = MakeWallet(wallet); return status; } @@ -282,7 +285,7 @@ public: } std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override { - return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::unique_ptr<Wallet>& wallet) { fn(std::move(wallet)); })); + return HandleLoadWallet(std::move(fn)); } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { @@ -312,10 +315,11 @@ public: return MakeHandler( ::uiInterface.NotifyHeaderTip_connect([fn](bool initial_download, const CBlockIndex* block) { fn(initial_download, block->nHeight, block->GetBlockTime(), - GuessVerificationProgress(Params().TxData(), block)); + /* verification progress is unused when a header was received */ 0); })); } - InitInterfaces m_interfaces; + NodeContext* context() override { return &m_context; } + NodeContext m_context; }; } // namespace diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 4ee467014c..38aeb06324 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -1,13 +1,13 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H -#include <addrdb.h> // For banmap_t #include <amount.h> // For CAmount #include <net.h> // For CConnman::NumConnections +#include <net_types.h> // For banmap_t #include <netaddress.h> // For Network #include <support/allocators/secure.h> // For SecureString @@ -28,6 +28,7 @@ class RPCTimerInterface; class UniValue; class proxyType; struct CNodeStateStats; +struct NodeContext; enum class WalletCreationStatus; namespace interfaces { @@ -77,7 +78,7 @@ public: virtual void initParameterInteraction() = 0; //! Get warnings. - virtual std::string getWarnings(const std::string& type) = 0; + virtual std::string getWarnings() = 0; // Get log flags. virtual uint32_t getLogCategories() = 0; @@ -254,6 +255,9 @@ public: using NotifyHeaderTipFn = std::function<void(bool initial_download, int height, int64_t block_time, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) = 0; + + //! Return pointer to internal chain interface, useful for testing. + virtual NodeContext* context() { return nullptr; } }; //! Return implementation of Node interface. diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 9b0a8b64c9..baea71d0bb 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,8 +18,9 @@ #include <wallet/feebumper.h> #include <wallet/fees.h> #include <wallet/ismine.h> -#include <wallet/rpcwallet.h> #include <wallet/load.h> +#include <wallet/psbtwallet.h> +#include <wallet/rpcwallet.h> #include <wallet/wallet.h> #include <memory> @@ -31,7 +32,7 @@ namespace interfaces { namespace { //! Construct wallet tx struct. -WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, const CWalletTx& wtx) +WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) { WalletTx result; result.tx = wtx.tx; @@ -46,10 +47,10 @@ WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, co result.txout_is_mine.emplace_back(wallet.IsMine(txout)); result.txout_address.emplace_back(); result.txout_address_is_mine.emplace_back(ExtractDestination(txout.scriptPubKey, result.txout_address.back()) ? - IsMine(wallet, result.txout_address.back()) : + wallet.IsMine(result.txout_address.back()) : ISMINE_NO); } - result.credit = wtx.GetCredit(locked_chain, ISMINE_ALL); + result.credit = wtx.GetCredit(ISMINE_ALL); result.debit = wtx.GetDebit(ISMINE_ALL); result.change = wtx.GetChange(); result.time = wtx.GetTxTime(); @@ -63,21 +64,20 @@ WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const C { WalletTxStatus result; result.block_height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock).get_value_or(std::numeric_limits<int>::max()); - result.blocks_to_maturity = wtx.GetBlocksToMaturity(locked_chain); - result.depth_in_main_chain = wtx.GetDepthInMainChain(locked_chain); + result.blocks_to_maturity = wtx.GetBlocksToMaturity(); + result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; result.is_final = locked_chain.checkFinalTx(*wtx.tx); result.is_trusted = wtx.IsTrusted(locked_chain); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); - result.is_in_main_chain = wtx.IsInMainChain(locked_chain); + result.is_in_main_chain = wtx.IsInMainChain(); return result; } //! Construct wallet TxOut struct. -WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, - CWallet& wallet, +WalletTxOut MakeWalletTxOut(CWallet& wallet, const CWalletTx& wtx, int n, int depth) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) @@ -86,7 +86,7 @@ WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, result.txout = wtx.tx->vout[n]; result.time = wtx.GetTxTime(); result.depth_in_main_chain = depth; - result.is_spent = wallet.IsSpent(locked_chain, wtx.GetHash(), n); + result.is_spent = wallet.IsSpent(wtx.GetHash(), n); return result; } @@ -117,10 +117,31 @@ public: std::string error; return m_wallet->GetNewDestination(type, label, dest, error); } - bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet->GetPubKey(address, pub_key); } - bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet->GetKey(address, key); } - bool isSpendable(const CTxDestination& dest) override { return IsMine(*m_wallet, dest) & ISMINE_SPENDABLE; } - bool haveWatchOnly() override { return m_wallet->HaveWatchOnly(); }; + bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) override + { + std::unique_ptr<SigningProvider> provider = m_wallet->GetSigningProvider(script); + if (provider) { + return provider->GetPubKey(address, pub_key); + } + return false; + } + bool getPrivKey(const CScript& script, const CKeyID& address, CKey& key) override + { + std::unique_ptr<SigningProvider> provider = m_wallet->GetSigningProvider(script); + if (provider) { + return provider->GetKey(address, key); + } + return false; + } + bool isSpendable(const CTxDestination& dest) override { return m_wallet->IsMine(dest) & ISMINE_SPENDABLE; } + bool haveWatchOnly() override + { + auto spk_man = m_wallet->GetLegacyScriptPubKeyMan(); + if (spk_man) { + return spk_man->HaveWatchOnly(); + } + return false; + }; bool setAddressBook(const CTxDestination& dest, const std::string& name, const std::string& purpose) override { return m_wallet->SetAddressBook(dest, name, purpose); @@ -143,7 +164,7 @@ public: *name = it->second.name; } if (is_mine) { - *is_mine = IsMine(*m_wallet, dest); + *is_mine = m_wallet->IsMine(dest); } if (purpose) { *purpose = it->second.purpose; @@ -155,20 +176,21 @@ public: LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; for (const auto& item : m_wallet->mapAddressBook) { - result.emplace_back(item.first, IsMine(*m_wallet, item.first), item.second.name, item.second.purpose); + result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.name, item.second.purpose); } return result; } - void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->LearnRelatedScripts(key, type); } bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { LOCK(m_wallet->cs_wallet); - return m_wallet->AddDestData(dest, key, value); + WalletBatch batch{m_wallet->GetDatabase()}; + return m_wallet->AddDestData(batch, dest, key, value); } bool eraseDestData(const CTxDestination& dest, const std::string& key) override { LOCK(m_wallet->cs_wallet); - return m_wallet->EraseDestData(dest, key); + WalletBatch batch{m_wallet->GetDatabase()}; + return m_wallet->EraseDestData(batch, dest, key); } std::vector<std::string> getDestValues(const std::string& prefix) override { @@ -228,7 +250,7 @@ public: { auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); - return m_wallet->AbandonTransaction(*locked_chain, txid); + return m_wallet->AbandonTransaction(txid); } bool transactionCanBeBumped(const uint256& txid) override { @@ -275,7 +297,7 @@ public: LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { - return MakeWalletTx(*locked_chain, *m_wallet, mi->second); + return MakeWalletTx(*m_wallet, mi->second); } return {}; } @@ -286,7 +308,7 @@ public: std::vector<WalletTx> result; result.reserve(m_wallet->mapWallet.size()); for (const auto& entry : m_wallet->mapWallet) { - result.emplace_back(MakeWalletTx(*locked_chain, *m_wallet, entry.second)); + result.emplace_back(MakeWalletTx(*m_wallet, entry.second)); } return result; } @@ -331,10 +353,18 @@ public: in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; tx_status = MakeWalletTxStatus(*locked_chain, mi->second); - return MakeWalletTx(*locked_chain, *m_wallet, mi->second); + return MakeWalletTx(*m_wallet, mi->second); } return {}; } + TransactionError fillPSBT(PartiallySignedTransaction& psbtx, + bool& complete, + int sighash_type = 1 /* SIGHASH_ALL */, + bool sign = true, + bool bip32derivs = false) override + { + return FillPSBT(m_wallet.get(), psbtx, complete, sighash_type, sign, bip32derivs); + } WalletBalances getBalances() override { const auto bal = m_wallet->GetBalance(); @@ -342,7 +372,7 @@ public: result.balance = bal.m_mine_trusted; result.unconfirmed_balance = bal.m_mine_untrusted_pending; result.immature_balance = bal.m_mine_immature; - result.have_watch_only = m_wallet->HaveWatchOnly(); + result.have_watch_only = haveWatchOnly(); if (result.have_watch_only) { result.watch_only_balance = bal.m_watchonly_trusted; result.unconfirmed_watch_only_balance = bal.m_watchonly_untrusted_pending; @@ -400,7 +430,7 @@ public: auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*locked_chain, *m_wallet, *coin.tx, coin.i, coin.nDepth)); + MakeWalletTxOut(*m_wallet, *coin.tx, coin.i, coin.nDepth)); } } return result; @@ -415,9 +445,9 @@ public: result.emplace_back(); auto it = m_wallet->mapWallet.find(output.hash); if (it != m_wallet->mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(*locked_chain); + int depth = it->second.GetDepthInMainChain(); if (depth >= 0) { - result.back() = MakeWalletTxOut(*locked_chain, *m_wallet, it->second, output.n, depth); + result.back() = MakeWalletTxOut(*m_wallet, it->second, output.n, depth); } } } @@ -489,7 +519,11 @@ public: : m_chain(chain), m_wallet_filenames(std::move(wallet_filenames)) { } - void registerRpcs() override { return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); } + void registerRpcs() override + { + g_rpc_chain = &m_chain; + return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); + } bool verify() override { return VerifyWallets(m_chain, m_wallet_filenames); } bool load() override { return LoadWallets(m_chain, m_wallet_filenames); } void start(CScheduler& scheduler) override { return StartWallets(scheduler); } diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index a96b93b4c3..d4280e8091 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ #include <functional> #include <map> #include <memory> +#include <psbt.h> #include <stdint.h> #include <string> #include <tuple> @@ -81,10 +82,10 @@ public: virtual bool getNewDestination(const OutputType type, const std::string label, CTxDestination& dest) = 0; //! Get public key. - virtual bool getPubKey(const CKeyID& address, CPubKey& pub_key) = 0; + virtual bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) = 0; //! Get private key. - virtual bool getPrivKey(const CKeyID& address, CKey& key) = 0; + virtual bool getPrivKey(const CScript& script, const CKeyID& address, CKey& key) = 0; //! Return whether wallet has private key. virtual bool isSpendable(const CTxDestination& dest) = 0; @@ -107,10 +108,6 @@ public: //! Get wallet address list. virtual std::vector<WalletAddress> getAddresses() = 0; - //! Add scripts to key store so old so software versions opening the wallet - //! database can detect payments to newer address types. - virtual void learnRelatedScripts(const CPubKey& key, OutputType type) = 0; - //! Add dest data. virtual bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) = 0; @@ -194,6 +191,13 @@ public: bool& in_mempool, int& num_blocks) = 0; + //! Fill PSBT. + virtual TransactionError fillPSBT(PartiallySignedTransaction& psbtx, + bool& complete, + int sighash_type = 1 /* SIGHASH_ALL */, + bool sign = true, + bool bip32derivs = false) = 0; + //! Get balances. virtual WalletBalances getBalances() = 0; diff --git a/src/key.cpp b/src/key.cpp index 3ba21753a2..b6ed29e8e3 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -83,13 +83,13 @@ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *ou * <http://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are * included. * - * privkey must point to an output buffer of length at least CKey::PRIVATE_KEY_SIZE bytes. + * privkey must point to an output buffer of length at least CKey::SIZE bytes. * privkeylen must initially be set to the size of the privkey buffer. Upon return it * will be set to the number of bytes used in the buffer. * key32 must point to a 32-byte raw private key. */ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *privkey, size_t *privkeylen, const unsigned char *key32, bool compressed) { - assert(*privkeylen >= CKey::PRIVATE_KEY_SIZE); + assert(*privkeylen >= CKey::SIZE); secp256k1_pubkey pubkey; size_t pubkeylen = 0; if (!secp256k1_ec_pubkey_create(ctx, &pubkey, key32)) { @@ -115,11 +115,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = CPubKey::COMPRESSED_PUBLIC_KEY_SIZE; + pubkeylen = CPubKey::COMPRESSED_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; - assert(*privkeylen == CKey::COMPRESSED_PRIVATE_KEY_SIZE); + assert(*privkeylen == CKey::COMPRESSED_SIZE); } else { static const unsigned char begin[] = { 0x30,0x82,0x01,0x13,0x02,0x01,0x01,0x04,0x20 @@ -141,11 +141,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = CPubKey::PUBLIC_KEY_SIZE; + pubkeylen = CPubKey::SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_UNCOMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; - assert(*privkeylen == CKey::PRIVATE_KEY_SIZE); + assert(*privkeylen == CKey::SIZE); } return 1; } @@ -173,8 +173,8 @@ CPrivKey CKey::GetPrivKey() const { CPrivKey privkey; int ret; size_t privkeylen; - privkey.resize(PRIVATE_KEY_SIZE); - privkeylen = PRIVATE_KEY_SIZE; + privkey.resize(SIZE); + privkeylen = SIZE; ret = ec_privkey_export_der(secp256k1_context_sign, privkey.data(), &privkeylen, begin(), fCompressed); assert(ret); privkey.resize(privkeylen); @@ -184,7 +184,7 @@ CPrivKey CKey::GetPrivKey() const { CPubKey CKey::GetPubKey() const { assert(fValid); secp256k1_pubkey pubkey; - size_t clen = CPubKey::PUBLIC_KEY_SIZE; + size_t clen = CPubKey::SIZE; CPubKey result; int ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pubkey, begin()); assert(ret); @@ -276,7 +276,7 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); - assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); + assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { assert(size() == 32); @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,7 +19,7 @@ /** * secure_allocator is defined in allocators.h * CPrivKey is a serialized private key, with all parameters included - * (PRIVATE_KEY_SIZE bytes) + * (SIZE bytes) */ typedef std::vector<unsigned char, secure_allocator<unsigned char> > CPrivKey; @@ -30,15 +30,15 @@ public: /** * secp256k1: */ - static const unsigned int PRIVATE_KEY_SIZE = 279; - static const unsigned int COMPRESSED_PRIVATE_KEY_SIZE = 214; + static const unsigned int SIZE = 279; + static const unsigned int COMPRESSED_SIZE = 214; /** * see www.keylength.com * script supports up to 75 for single byte push */ static_assert( - PRIVATE_KEY_SIZE >= COMPRESSED_PRIVATE_KEY_SIZE, - "COMPRESSED_PRIVATE_KEY_SIZE is larger than PRIVATE_KEY_SIZE"); + SIZE >= COMPRESSED_SIZE, + "COMPRESSED_SIZE is larger than SIZE"); private: //! Whether this private key is valid. We check for correctness when modifying the key diff --git a/src/key_io.cpp b/src/key_io.cpp index 363055d6b3..d2f5be93f5 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -73,7 +73,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par { std::vector<unsigned char> data; uint160 hash; - if (DecodeBase58Check(str, data)) { + if (DecodeBase58Check(str, data, 21)) { // base58-encoded Bitcoin addresses. // Public-key-hash-addresses have version 0 (or 111 testnet). // The data vector contains RIPEMD160(SHA256(pubkey)), where pubkey is the serialized public key. @@ -133,7 +133,7 @@ CKey DecodeSecret(const std::string& str) { CKey key; std::vector<unsigned char> data; - if (DecodeBase58Check(str, data)) { + if (DecodeBase58Check(str, data, 34)) { const std::vector<unsigned char>& privkey_prefix = Params().Base58Prefix(CChainParams::SECRET_KEY); if ((data.size() == 32 + privkey_prefix.size() || (data.size() == 33 + privkey_prefix.size() && data.back() == 1)) && std::equal(privkey_prefix.begin(), privkey_prefix.end(), data.begin())) { @@ -164,7 +164,7 @@ CExtPubKey DecodeExtPubKey(const std::string& str) { CExtPubKey key; std::vector<unsigned char> data; - if (DecodeBase58Check(str, data)) { + if (DecodeBase58Check(str, data, 78)) { const std::vector<unsigned char>& prefix = Params().Base58Prefix(CChainParams::EXT_PUBLIC_KEY); if (data.size() == BIP32_EXTKEY_SIZE + prefix.size() && std::equal(prefix.begin(), prefix.end(), data.begin())) { key.Decode(data.data() + prefix.size()); @@ -187,7 +187,7 @@ CExtKey DecodeExtKey(const std::string& str) { CExtKey key; std::vector<unsigned char> data; - if (DecodeBase58Check(str, data)) { + if (DecodeBase58Check(str, data, 78)) { const std::vector<unsigned char>& prefix = Params().Base58Prefix(CChainParams::EXT_SECRET_KEY); if (data.size() == BIP32_EXTKEY_SIZE + prefix.size() && std::equal(prefix.begin(), prefix.end(), data.begin())) { key.Decode(data.data() + prefix.size()); diff --git a/src/logging.cpp b/src/logging.cpp index 60ab486198..6fd916b603 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -67,6 +67,9 @@ bool BCLog::Logger::StartLogging() if (m_print_to_file) FileWriteStr(s, m_fileout); if (m_print_to_console) fwrite(s.data(), 1, s.size(), stdout); + for (const auto& cb : m_print_callbacks) { + cb(s); + } m_msgs_before_open.pop_front(); } @@ -81,6 +84,7 @@ void BCLog::Logger::DisconnectTestLogger() m_buffering = true; if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; + m_print_callbacks.clear(); } void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) @@ -91,7 +95,15 @@ void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) bool BCLog::Logger::EnableCategory(const std::string& str) { BCLog::LogFlags flag; - if (!GetLogCategory(flag, str)) return false; + if (!GetLogCategory(flag, str)) { + if (str == "db") { + // DEPRECATION: Added in 0.20, should start returning an error in 0.21 + LogPrintf("Warning: logging category 'db' is deprecated, use 'walletdb' instead\n"); + EnableCategory(BCLog::WALLETDB); + return true; + } + return false; + } EnableCategory(flag); return true; } @@ -135,7 +147,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::HTTP, "http"}, {BCLog::BENCH, "bench"}, {BCLog::ZMQ, "zmq"}, - {BCLog::DB, "db"}, + {BCLog::WALLETDB, "walletdb"}, {BCLog::RPC, "rpc"}, {BCLog::ESTIMATEFEE, "estimatefee"}, {BCLog::ADDRMAN, "addrman"}, @@ -150,6 +162,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::COINDB, "coindb"}, {BCLog::QT, "qt"}, {BCLog::LEVELDB, "leveldb"}, + {BCLog::VALIDATION, "validation"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -270,6 +283,9 @@ void BCLog::Logger::LogPrintStr(const std::string& str) fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } + for (const auto& cb : m_print_callbacks) { + cb(str_prefixed); + } if (m_print_to_file) { assert(m_fileout != nullptr); diff --git a/src/logging.h b/src/logging.h index 75cd5353c0..b2fde1b9ea 100644 --- a/src/logging.h +++ b/src/logging.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,7 +39,7 @@ namespace BCLog { HTTP = (1 << 3), BENCH = (1 << 4), ZMQ = (1 << 5), - DB = (1 << 6), + WALLETDB = (1 << 6), RPC = (1 << 7), ESTIMATEFEE = (1 << 8), ADDRMAN = (1 << 9), @@ -54,6 +54,7 @@ namespace BCLog { COINDB = (1 << 18), QT = (1 << 19), LEVELDB = (1 << 20), + VALIDATION = (1 << 21), ALL = ~(uint32_t)0, }; @@ -77,6 +78,9 @@ namespace BCLog { std::string LogTimestampStr(const std::string& str); + /** Slots that connect to the print signal */ + std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {}; + public: bool m_print_to_console = false; bool m_print_to_file = false; @@ -95,7 +99,22 @@ namespace BCLog { bool Enabled() const { std::lock_guard<std::mutex> scoped_lock(m_cs); - return m_buffering || m_print_to_console || m_print_to_file; + return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); + } + + /** Connect a slot to the print signal and return the connection */ + std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.push_back(std::move(fun)); + return --m_print_callbacks.end(); + } + + /** Delete a connection */ + void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.erase(it); } /** Start logging (and flush all buffered messages) */ @@ -155,12 +174,13 @@ static inline void LogPrintf(const char* fmt, const Args&... args) } } -template <typename... Args> -static inline void LogPrint(const BCLog::LogFlags& category, const Args&... args) -{ - if (LogAcceptCategory((category))) { - LogPrintf(args...); - } -} +// Use a macro instead of a function for conditional logging to prevent +// evaluating arguments when logging for the category is not enabled. +#define LogPrint(category, ...) \ + do { \ + if (LogAcceptCategory((category))) { \ + LogPrintf(__VA_ARGS__); \ + } \ + } while (0) #endif // BITCOIN_LOGGING_H diff --git a/src/logging/timer.h b/src/logging/timer.h new file mode 100644 index 0000000000..45bfc4aa65 --- /dev/null +++ b/src/logging/timer.h @@ -0,0 +1,104 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_LOGGING_TIMER_H +#define BITCOIN_LOGGING_TIMER_H + +#include <logging.h> +#include <util/macros.h> +#include <util/time.h> + +#include <chrono> +#include <string> + + +namespace BCLog { + +//! RAII-style object that outputs timing information to logs. +template <typename TimeType> +class Timer +{ +public: + //! If log_category is left as the default, end_msg will log unconditionally + //! (instead of being filtered by category). + Timer( + std::string prefix, + std::string end_msg, + BCLog::LogFlags log_category = BCLog::LogFlags::ALL) : + m_prefix(std::move(prefix)), + m_title(std::move(end_msg)), + m_log_category(log_category) + { + this->Log(strprintf("%s started", m_title)); + m_start_t = GetTime<std::chrono::microseconds>(); + } + + ~Timer() + { + this->Log(strprintf("%s completed", m_title)); + } + + void Log(const std::string& msg) + { + const std::string full_msg = this->LogMsg(msg); + + if (m_log_category == BCLog::LogFlags::ALL) { + LogPrintf("%s\n", full_msg); + } else { + LogPrint(m_log_category, "%s\n", full_msg); + } + } + + std::string LogMsg(const std::string& msg) + { + const auto end_time = GetTime<std::chrono::microseconds>() - m_start_t; + if (m_start_t.count() <= 0) { + return strprintf("%s: %s", m_prefix, msg); + } + + std::string units = ""; + float divisor = 1; + + if (std::is_same<TimeType, std::chrono::microseconds>::value) { + units = "μs"; + } else if (std::is_same<TimeType, std::chrono::milliseconds>::value) { + units = "ms"; + divisor = 1000.; + } else if (std::is_same<TimeType, std::chrono::seconds>::value) { + units = "s"; + divisor = 1000. * 1000.; + } + + const float time_ms = end_time.count() / divisor; + return strprintf("%s: %s (%.2f%s)", m_prefix, msg, time_ms, units); + } + +private: + std::chrono::microseconds m_start_t{}; + + //! Log prefix; usually the name of the function this was created in. + const std::string m_prefix{}; + + //! A descriptive message of what is being timed. + const std::string m_title{}; + + //! Forwarded on to LogPrint if specified - has the effect of only + //! outputing the timing log when a particular debug= category is specified. + const BCLog::LogFlags m_log_category{}; + +}; + +} // namespace BCLog + + +#define LOG_TIME_MICROS(end_msg, ...) \ + BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_MILLIS(end_msg, ...) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_SECONDS(end_msg, ...) \ + BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) + + +#endif // BITCOIN_LOGGING_TIMER_H diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 052aebbc80..4ac6219886 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/miner.cpp b/src/miner.cpp index 4f51be8a08..6f4e10b6ed 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -45,7 +45,9 @@ BlockAssembler::Options::Options() { nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; } -BlockAssembler::BlockAssembler(const CChainParams& params, const Options& options) : chainparams(params) +BlockAssembler::BlockAssembler(const CTxMemPool& mempool, const CChainParams& params, const Options& options) + : chainparams(params), + m_mempool(mempool) { blockMinFeeRate = options.blockMinFeeRate; // Limit weight to between 4K and MAX_BLOCK_WEIGHT-4K for sanity: @@ -67,7 +69,8 @@ static BlockAssembler::Options DefaultOptions() return options; } -BlockAssembler::BlockAssembler(const CChainParams& params) : BlockAssembler(params, DefaultOptions()) {} +BlockAssembler::BlockAssembler(const CTxMemPool& mempool, const CChainParams& params) + : BlockAssembler(mempool, params, DefaultOptions()) {} void BlockAssembler::resetBlock() { @@ -103,7 +106,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate->vTxFees.push_back(-1); // updated at end pblocktemplate->vTxSigOpsCost.push_back(-1); // updated at end - LOCK2(cs_main, mempool.cs); + LOCK2(cs_main, m_mempool.cs); CBlockIndex* pindexPrev = ::ChainActive().Tip(); assert(pindexPrev != nullptr); nHeight = pindexPrev->nHeight + 1; @@ -162,7 +165,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblock->nNonce = 0; pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); - CValidationState state; + BlockValidationState state; if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); } @@ -236,7 +239,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already int nDescendantsUpdated = 0; for (CTxMemPool::txiter it : alreadyAdded) { CTxMemPool::setEntries descendants; - mempool.CalculateDescendants(it, descendants); + m_mempool.CalculateDescendants(it, descendants); // Insert all descendants (not yet in block) into the modified set for (CTxMemPool::txiter desc : descendants) { if (alreadyAdded.count(desc)) @@ -268,7 +271,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already // cached size/sigops/fee values that are not actually correct. bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) { - assert (it != mempool.mapTx.end()); + assert(it != m_mempool.mapTx.end()); return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); } @@ -305,7 +308,7 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda // and modifying them for their already included ancestors UpdatePackagesForAdded(inBlock, mapModifiedTx); - CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); + CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = m_mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; // Limit the number of attempts to add transactions to the block when it is @@ -314,11 +317,10 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda const int64_t MAX_CONSECUTIVE_FAILURES = 1000; int64_t nConsecutiveFailed = 0; - while (mi != mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty()) - { + while (mi != m_mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty()) { // First try to find a new transaction in mapTx to evaluate. - if (mi != mempool.mapTx.get<ancestor_score>().end() && - SkipMapTxEntry(mempool.mapTx.project<0>(mi), mapModifiedTx, failedTx)) { + if (mi != m_mempool.mapTx.get<ancestor_score>().end() && + SkipMapTxEntry(m_mempool.mapTx.project<0>(mi), mapModifiedTx, failedTx)) { ++mi; continue; } @@ -328,13 +330,13 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda bool fUsingModified = false; modtxscoreiter modit = mapModifiedTx.get<ancestor_score>().begin(); - if (mi == mempool.mapTx.get<ancestor_score>().end()) { + if (mi == m_mempool.mapTx.get<ancestor_score>().end()) { // We're out of entries in mapTx; use the entry from mapModifiedTx iter = modit->iter; fUsingModified = true; } else { // Try to compare the mapTx entry to the mapModifiedTx entry - iter = mempool.mapTx.project<0>(mi); + iter = m_mempool.mapTx.project<0>(mi); if (modit != mapModifiedTx.get<ancestor_score>().end() && CompareTxMemPoolEntryByAncestorFee()(*modit, CTxMemPoolModifiedEntry(iter))) { // The best entry in mapModifiedTx has higher score @@ -389,7 +391,7 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda CTxMemPool::setEntries ancestors; uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - mempool.CalculateMemPoolAncestors(*iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); + m_mempool.CalculateMemPoolAncestors(*iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); onlyUnconfirmed(ancestors); ancestors.insert(iter); @@ -435,7 +437,7 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned ++nExtraNonce; unsigned int nHeight = pindexPrev->nHeight+1; // Height first in coinbase required for block.version=2 CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.vin[0].scriptSig = (CScript() << nHeight << CScriptNum(nExtraNonce)) + COINBASE_FLAGS; + txCoinbase.vin[0].scriptSig = (CScript() << nHeight << CScriptNum(nExtraNonce)); assert(txCoinbase.vin[0].scriptSig.size() <= 100); pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); diff --git a/src/miner.h b/src/miner.h index 7c4c455072..cc8fc31a9f 100644 --- a/src/miner.h +++ b/src/miner.h @@ -147,6 +147,7 @@ private: int nHeight; int64_t nLockTimeCutoff; const CChainParams& chainparams; + const CTxMemPool& m_mempool; public: struct Options { @@ -155,8 +156,8 @@ public: CFeeRate blockMinFeeRate; }; - explicit BlockAssembler(const CChainParams& params); - BlockAssembler(const CChainParams& params, const Options& options); + explicit BlockAssembler(const CTxMemPool& mempool, const CChainParams& params); + explicit BlockAssembler(const CTxMemPool& mempool, const CChainParams& params, const Options& options); /** Construct a new block template with coinbase to scriptPubKeyIn */ std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); @@ -175,7 +176,7 @@ private: /** Add transactions based on feerate including unconfirmed ancestors * Increments nPackagesSelected / nDescendantsUpdated with corresponding * statistics from the package selection (for logging statistics). */ - void addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + void addPackageTxs(int& nPackagesSelected, int& nDescendantsUpdated) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); // helper functions for addPackageTxs() /** Remove confirmed (inBlock) entries from given set */ @@ -189,13 +190,13 @@ private: bool TestPackageTransactions(const CTxMemPool::setEntries& package); /** Return true if given transaction from mapTx has already been evaluated, * or if the transaction's cached data in mapTx is incorrect. */ - bool SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + bool SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set& mapModifiedTx, CTxMemPool::setEntries& failedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); /** Sort the package in an order that is valid to appear in a block */ void SortForBlock(const CTxMemPool::setEntries& package, std::vector<CTxMemPool::txiter>& sortedEntries); /** Add descendants of given transactions to mapModifiedTx with ancestor * state updated assuming given transactions are inBlock. Returns number * of updated descendants. */ - int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set& mapModifiedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); }; /** Modify the extranonce in a block */ diff --git a/src/net.cpp b/src/net.cpp index c0b39925c6..9cd2d30d9d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,6 +16,7 @@ #include <crypto/sha256.h> #include <netbase.h> #include <net_permissions.h> +#include <random.h> #include <scheduler.h> #include <ui_interface.h> #include <util/strencodings.h> @@ -84,7 +85,7 @@ static const uint64_t RANDOMIZER_ID_LOCALHOSTNONCE = 0xd93e69e2bbfa5735ULL; // S bool fDiscover = true; bool fListen = true; bool g_relay_txes = !DEFAULT_BLOCKSONLY; -CCriticalSection cs_mapLocalHost; +RecursiveMutex cs_mapLocalHost; std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; std::string strSubVersion; @@ -409,7 +410,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (hSocket == INVALID_SOCKET) { return nullptr; } - connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, &proxyConnectionFailed); + connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, proxyConnectionFailed); } else { // no proxy needed (none set for target network) hSocket = CreateSocket(addrConnect); @@ -431,7 +432,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo std::string host; int port = default_port; SplitHostPort(std::string(pszDest), port, host); - connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, nullptr); + bool proxyConnectionFailed; + connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, proxyConnectionFailed); } if (!connected) { CloseSocket(hSocket); @@ -445,6 +447,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false, block_relay_only); pnode->AddRef(); + // We're making a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); + return pnode; } @@ -493,12 +498,13 @@ void CNode::SetAddrLocal(const CService& addrLocalIn) { #undef X #define X(name) stats.name = name -void CNode::copyStats(CNodeStats &stats) +void CNode::copyStats(CNodeStats &stats, std::vector<bool> &m_asmap) { stats.nodeid = this->GetId(); X(nServices); X(addr); X(addrBind); + stats.m_mapped_as = addr.GetMappedAS(m_asmap); if (m_tx_relay != nullptr) { LOCK(m_tx_relay->cs_filter); stats.fRelayTxes = m_tx_relay->fRelayTxes; @@ -567,42 +573,28 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete nLastRecv = nTimeMicros / 1000000; nRecvBytes += nBytes; while (nBytes > 0) { - - // get current incomplete message, or create a new one - if (vRecvMsg.empty() || - vRecvMsg.back().complete()) - vRecvMsg.push_back(CNetMessage(Params().MessageStart(), SER_NETWORK, INIT_PROTO_VERSION)); - - CNetMessage& msg = vRecvMsg.back(); - // absorb network data - int handled; - if (!msg.in_data) - handled = msg.readHeader(pch, nBytes); - else - handled = msg.readData(pch, nBytes); - - if (handled < 0) - return false; - - if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); - return false; - } + int handled = m_deserializer->Read(pch, nBytes); + if (handled < 0) return false; pch += handled; nBytes -= handled; - if (msg.complete()) { + if (m_deserializer->Complete()) { + // decompose a transport agnostic CNetMessage from the deserializer + CNetMessage msg = m_deserializer->GetMessage(Params().MessageStart(), nTimeMicros); + //store received bytes per message command //to prevent a memory DOS, only allow valid commands - mapMsgCmdSize::iterator i = mapRecvBytesPerMsgCmd.find(msg.hdr.pchCommand); + mapMsgCmdSize::iterator i = mapRecvBytesPerMsgCmd.find(msg.m_command); if (i == mapRecvBytesPerMsgCmd.end()) i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); assert(i != mapRecvBytesPerMsgCmd.end()); - i->second += msg.hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + i->second += msg.m_raw_message_size; + + // push the message to the process queue, + vRecvMsg.push_back(std::move(msg)); - msg.nTime = nTimeMicros; complete = true; } } @@ -636,8 +628,7 @@ int CNode::GetSendVersion() const return nSendVersion; } - -int CNetMessage::readHeader(const char *pch, unsigned int nBytes) +int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) { // copy data to temporary parsing buffer unsigned int nRemaining = 24 - nHdrPos; @@ -658,9 +649,10 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) return -1; } - // reject messages larger than MAX_SIZE - if (hdr.nMessageSize > MAX_SIZE) + // reject messages larger than MAX_SIZE or MAX_PROTOCOL_MESSAGE_LENGTH + if (hdr.nMessageSize > MAX_SIZE || hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { return -1; + } // switch state to reading message data in_data = true; @@ -668,7 +660,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) return nCopy; } -int CNetMessage::readData(const char *pch, unsigned int nBytes) +int V1TransportDeserializer::readData(const char *pch, unsigned int nBytes) { unsigned int nRemaining = hdr.nMessageSize - nDataPos; unsigned int nCopy = std::min(nRemaining, nBytes); @@ -685,14 +677,47 @@ int CNetMessage::readData(const char *pch, unsigned int nBytes) return nCopy; } -const uint256& CNetMessage::GetMessageHash() const +const uint256& V1TransportDeserializer::GetMessageHash() const { - assert(complete()); + assert(Complete()); if (data_hash.IsNull()) hasher.Finalize(data_hash.begin()); return data_hash; } +CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) { + // decompose a single CNetMessage from the TransportDeserializer + CNetMessage msg(std::move(vRecv)); + + // store state about valid header, netmagic and checksum + msg.m_valid_header = hdr.IsValid(message_start); + msg.m_valid_netmagic = (memcmp(hdr.pchMessageStart, message_start, CMessageHeader::MESSAGE_START_SIZE) == 0); + uint256 hash = GetMessageHash(); + + // store command string, payload size + msg.m_command = hdr.GetCommand(); + msg.m_message_size = hdr.nMessageSize; + msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + + // We just received a message off the wire, harvest entropy from the time (and the message checksum) + RandAddEvent(ReadLE32(hash.begin())); + + msg.m_valid_checksum = (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) == 0); + if (!msg.m_valid_checksum) { + LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s\n", + SanitizeString(msg.m_command), msg.m_message_size, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + } + + // store receive time + msg.m_time = time; + + // reset the network deserializer (prepare for the next message) + Reset(); + return msg; +} + size_t CConnman::SocketSendData(CNode *pnode) const EXCLUSIVE_LOCKS_REQUIRED(pnode->cs_vSend) { auto it = pnode->vSendMsg.begin(); @@ -1001,6 +1026,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { LOCK(cs_vNodes); vNodes.push_back(pnode); } + + // We received a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); } void CConnman::DisconnectNodes() @@ -1344,9 +1372,9 @@ void CConnman::SocketHandler() size_t nSizeAdded = 0; auto it(pnode->vRecvMsg.begin()); for (; it != pnode->vRecvMsg.end(); ++it) { - if (!it->complete()) - break; - nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; + // vRecvMsg contains only completed CNetMessage + // the single possible partially deserialized message are held by TransportDeserializer + nSizeAdded += it->m_raw_message_size; } { LOCK(pnode->cs_vProcessMsg); @@ -1458,7 +1486,7 @@ static void ThreadMapPort() if (externalIPAddress[0]) { CNetAddr resolved; if (LookupHost(externalIPAddress, resolved, false)) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString().c_str()); + LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString()); AddLocal(resolved, LOCAL_UPNP); } } else { @@ -1583,7 +1611,7 @@ void CConnman::ThreadDNSAddressSeed() continue; } unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed - if (LookupHost(host.c_str(), vIPs, nMaxIPs, true)) { + if (LookupHost(host, vIPs, nMaxIPs, true)) { for (const CNetAddr& ip : vIPs) { int nOneDay = 24*3600; CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); @@ -1742,7 +1770,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // but inbound and addnode peers do not use our outbound slots. Inbound peers // also have the added issue that they're attacker controlled and could be used // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup()); + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); if (pnode->m_tx_relay == nullptr) { nOutboundBlockRelay++; } else if (!pnode->fFeeler) { @@ -1790,7 +1818,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup())) { + if (!fFeeler && setConnected.count(addr.GetGroup(addrman.m_asmap))) { break; } @@ -1881,7 +1909,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() } for (const std::string& strAddNode : lAddresses) { - CService service(LookupNumeric(strAddNode.c_str(), Params().GetDefaultPort())); + CService service(LookupNumeric(strAddNode, Params().GetDefaultPort())); AddedNodeInfo addedNode{strAddNode, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port @@ -2475,7 +2503,7 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) vstats.reserve(vNodes.size()); for (CNode* pnode : vNodes) { vstats.emplace_back(); - pnode->copyStats(vstats.back()); + pnode->copyStats(vstats.back(), addrman.m_asmap); } } @@ -2650,11 +2678,10 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn addrBind(addrBindIn), fInbound(fInboundIn), nKeyedNetGroup(nKeyedNetGroupIn), - addrKnown(5000, 0.001), // Don't relay addr messages to peers that we connect to as block-relay-only // peers (to prevent adversaries from inferring these links from addr // traffic). - m_addr_relay_peer(!block_relay_only), + m_addr_known{block_relay_only ? nullptr : MakeUnique<CRollingBloomFilter>(5000, 0.001)}, id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), @@ -2676,6 +2703,8 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn } else { LogPrint(BCLog::NET, "Added connection peer=%d\n", id); } + + m_deserializer = MakeUnique<V1TransportDeserializer>(V1TransportDeserializer(Params().MessageStart(), SER_NETWORK, INIT_PROTO_VERSION)); } CNode::~CNode() @@ -2692,7 +2721,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->GetId()); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command), nMessageSize, pnode->GetId()); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -2761,7 +2790,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup()); + std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.m_asmap)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -153,6 +153,7 @@ public: bool m_use_addrman_outgoing = true; std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; + std::vector<bool> m_asmap; }; void Init(const Options& connOptions) { @@ -330,6 +331,8 @@ public: */ int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = asmap; } + private: struct ListenSocket { public: @@ -384,10 +387,10 @@ private: static bool NodeFullyConnected(const CNode* pnode); // Network usage totals - CCriticalSection cs_totalBytesRecv; - CCriticalSection cs_totalBytesSent; - uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv); - uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent); + RecursiveMutex cs_totalBytesRecv; + RecursiveMutex cs_totalBytesSent; + uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv) {0}; + uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; // outbound limit & stats uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent); @@ -410,12 +413,12 @@ private: bool fAddressesInitialized{false}; CAddrMan addrman; std::deque<std::string> vOneShots GUARDED_BY(cs_vOneShots); - CCriticalSection cs_vOneShots; + RecursiveMutex cs_vOneShots; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); - CCriticalSection cs_vAddedNodes; + RecursiveMutex cs_vAddedNodes; std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); std::list<CNode*> vNodesDisconnected; - mutable CCriticalSection cs_vNodes; + mutable RecursiveMutex cs_vNodes; std::atomic<NodeId> nLastNodeId{0}; unsigned int nPrevNodeCount{0}; @@ -480,8 +483,6 @@ private: friend struct CConnmanTest; }; -extern std::unique_ptr<CConnman> g_connman; -extern std::unique_ptr<BanMan> g_banman; void Discover(); void StartMapPort(); void InterruptMapPort(); @@ -567,7 +568,7 @@ struct LocalServiceInfo { int nPort; }; -extern CCriticalSection cs_mapLocalHost; +extern RecursiveMutex cs_mapLocalHost; extern std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); extern const std::string NET_MESSAGE_COMMAND_OTHER; @@ -605,60 +606,110 @@ public: CAddress addr; // Bind address of our side of the connection CAddress addrBind; + uint32_t m_mapped_as; }; - +/** Transport protocol agnostic message container. + * Ideally it should only contain receive time, payload, + * command and size. + */ class CNetMessage { +public: + CDataStream m_recv; // received message data + int64_t m_time = 0; // time (in microseconds) of message receipt. + bool m_valid_netmagic = false; + bool m_valid_header = false; + bool m_valid_checksum = false; + uint32_t m_message_size = 0; // size of the payload + uint32_t m_raw_message_size = 0; // used wire size of the message (including header/checksum) + std::string m_command; + + CNetMessage(CDataStream&& recv_in) : m_recv(std::move(recv_in)) {} + + void SetVersion(int nVersionIn) + { + m_recv.SetVersion(nVersionIn); + } +}; + +/** The TransportDeserializer takes care of holding and deserializing the + * network receive buffer. It can deserialize the network buffer into a + * transport protocol agnostic CNetMessage (command & payload) + */ +class TransportDeserializer { +public: + // returns true if the current deserialization is complete + virtual bool Complete() const = 0; + // set the serialization context version + virtual void SetVersion(int version) = 0; + // read and deserialize data + virtual int Read(const char *data, unsigned int bytes) = 0; + // decomposes a message from the context + virtual CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) = 0; + virtual ~TransportDeserializer() {} +}; + +class V1TransportDeserializer final : public TransportDeserializer +{ private: mutable CHash256 hasher; mutable uint256 data_hash; -public: bool in_data; // parsing header (false) or data (true) - CDataStream hdrbuf; // partially received header CMessageHeader hdr; // complete header - unsigned int nHdrPos; - CDataStream vRecv; // received message data + unsigned int nHdrPos; unsigned int nDataPos; - int64_t nTime; // time (in microseconds) of message receipt. + const uint256& GetMessageHash() const; + int readHeader(const char *pch, unsigned int nBytes); + int readData(const char *pch, unsigned int nBytes); - CNetMessage(const CMessageHeader::MessageStartChars& pchMessageStartIn, int nTypeIn, int nVersionIn) : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) { + void Reset() { + vRecv.clear(); + hdrbuf.clear(); hdrbuf.resize(24); in_data = false; nHdrPos = 0; nDataPos = 0; - nTime = 0; + data_hash.SetNull(); + hasher.Reset(); + } + +public: + + V1TransportDeserializer(const CMessageHeader::MessageStartChars& pchMessageStartIn, int nTypeIn, int nVersionIn) : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) { + Reset(); } - bool complete() const + bool Complete() const override { if (!in_data) return false; return (hdr.nMessageSize == nDataPos); } - - const uint256& GetMessageHash() const; - - void SetVersion(int nVersionIn) + void SetVersion(int nVersionIn) override { hdrbuf.SetVersion(nVersionIn); vRecv.SetVersion(nVersionIn); } - - int readHeader(const char *pch, unsigned int nBytes); - int readData(const char *pch, unsigned int nBytes); + int Read(const char *pch, unsigned int nBytes) override { + int ret = in_data ? readData(pch, nBytes) : readHeader(pch, nBytes); + if (ret < 0) Reset(); + return ret; + } + CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) override; }; - /** Information about a peer */ class CNode { friend class CConnman; public: + std::unique_ptr<TransportDeserializer> m_deserializer; + // socket std::atomic<ServiceFlags> nServices{NODE_NONE}; SOCKET hSocket GUARDED_BY(cs_hSocket); @@ -666,15 +717,15 @@ public: size_t nSendOffset{0}; // offset inside the first vSendMsg already sent uint64_t nSendBytes GUARDED_BY(cs_vSend){0}; std::deque<std::vector<unsigned char>> vSendMsg GUARDED_BY(cs_vSend); - CCriticalSection cs_vSend; - CCriticalSection cs_hSocket; - CCriticalSection cs_vRecv; + RecursiveMutex cs_vSend; + RecursiveMutex cs_hSocket; + RecursiveMutex cs_vRecv; - CCriticalSection cs_vProcessMsg; + RecursiveMutex cs_vProcessMsg; std::list<CNetMessage> vProcessMsg GUARDED_BY(cs_vProcessMsg); size_t nProcessQueueSize{0}; - CCriticalSection cs_sendProcessing; + RecursiveMutex cs_sendProcessing; std::deque<CInv> vRecvGetData; uint64_t nRecvBytes GUARDED_BY(cs_vRecv){0}; @@ -729,23 +780,22 @@ public: // flood relay std::vector<CAddress> vAddrToSend; - CRollingBloomFilter addrKnown; + const std::unique_ptr<CRollingBloomFilter> m_addr_known; bool fGetAddr{false}; int64_t nNextAddrSend GUARDED_BY(cs_sendProcessing){0}; int64_t nNextLocalAddrSend GUARDED_BY(cs_sendProcessing){0}; - const bool m_addr_relay_peer; - bool IsAddrRelayPeer() const { return m_addr_relay_peer; } + bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } // List of block ids we still have announce. // There is no final sorting before sending, as they are always sent immediately // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); - CCriticalSection cs_inventory; + RecursiveMutex cs_inventory; struct TxRelay { TxRelay() { pfilter = MakeUnique<CBloomFilter>(); } - mutable CCriticalSection cs_filter; + mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs @@ -753,7 +803,7 @@ public: bool fRelayTxes GUARDED_BY(cs_filter){false}; std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter); - mutable CCriticalSection cs_tx_inventory; + mutable RecursiveMutex cs_tx_inventory; CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; // Set of transaction ids we still have to announce. // They are sorted by the mempool before relay, so the order is not important. @@ -762,9 +812,9 @@ public: bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; // Last time a "MEMPOOL" request was serviced. std::atomic<std::chrono::seconds> m_last_mempool_req{std::chrono::seconds{0}}; - int64_t nNextInvSend{0}; + std::chrono::microseconds nNextInvSend{0}; - CCriticalSection cs_feeFilter; + RecursiveMutex cs_feeFilter; // Minimum fee rate with which to filter inv's to this node CAmount minFeeFilter GUARDED_BY(cs_feeFilter){0}; CAmount lastSentFeeFilter{0}; @@ -826,12 +876,12 @@ private: NetPermissionFlags m_permissionFlags{ PF_NONE }; std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread - mutable CCriticalSection cs_addrName; + mutable RecursiveMutex cs_addrName; std::string addrName GUARDED_BY(cs_addrName); // Our address, as reported by the peer CService addrLocal GUARDED_BY(cs_addrLocal); - mutable CCriticalSection cs_addrLocal; + mutable RecursiveMutex cs_addrLocal; public: NodeId GetId() const { @@ -884,7 +934,8 @@ public: void AddAddressKnown(const CAddress& _addr) { - addrKnown.insert(_addr.GetKey()); + assert(m_addr_known); + m_addr_known->insert(_addr.GetKey()); } void PushAddress(const CAddress& _addr, FastRandomContext &insecure_rand) @@ -892,7 +943,8 @@ public: // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. - if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { + assert(m_addr_known); + if (_addr.IsValid() && !m_addr_known->contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { @@ -931,7 +983,7 @@ public: void CloseSocketDisconnect(); - void copyStats(CNodeStats &stats); + void copyStats(CNodeStats &stats, std::vector<bool> &m_asmap); ServiceFlags GetLocalServices() const { @@ -943,11 +995,13 @@ public: void MaybeSetAddrName(const std::string& addrNameIn); }; - - - - /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ int64_t PoissonNextSend(int64_t now, int average_interval_seconds); +/** Wrapper to return mockable type */ +inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, std::chrono::seconds average_interval) +{ + return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; +} + #endif // BITCOIN_NET_H diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index ef6c40ce20..22fa5ee73b 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -71,7 +71,7 @@ bool NetWhitebindPermissions::TryParse(const std::string str, NetWhitebindPermis const std::string strBind = str.substr(offset); CService addrBind; - if (!Lookup(strBind.c_str(), addrBind, 0, false)) { + if (!Lookup(strBind, addrBind, 0, false)) { error = ResolveErrMsg("whitebind", strBind); return false; } @@ -94,7 +94,7 @@ bool NetWhitelistPermissions::TryParse(const std::string str, NetWhitelistPermis const std::string net = str.substr(offset); CSubNet subnet; - LookupSubNet(net.c_str(), subnet); + LookupSubNet(net, subnet); if (!subnet.IsValid()) { error = strprintf(_("Invalid netmask specified in -whitelist: '%s'").translated, net); return false; diff --git a/src/net_permissions.h b/src/net_permissions.h index b3987de65f..a06d2f544d 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 4a3076e3c7..cf4aee0647 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,6 +29,7 @@ #include <util/validation.h> #include <memory> +#include <typeinfo> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." @@ -89,7 +90,7 @@ struct COrphanTx { int64_t nTimeExpire; size_t list_pos; }; -CCriticalSection g_cs_orphans; +RecursiveMutex g_cs_orphans; std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); void EraseOrphansFor(NodeId peer); @@ -147,6 +148,14 @@ namespace { std::unique_ptr<CRollingBloomFilter> recentRejects GUARDED_BY(cs_main); uint256 hashRecentRejectsChainTip GUARDED_BY(cs_main); + /* + * Filter for transactions that have been recently confirmed. + * We use this to avoid requesting transactions that have already been + * confirnmed. + */ + RecursiveMutex g_cs_recent_confirmed_transactions; + std::unique_ptr<CRollingBloomFilter> g_recent_confirmed_transactions GUARDED_BY(g_cs_recent_confirmed_transactions); + /** Blocks that are in flight, and that are in the queue to be downloaded. */ struct QueuedBlock { uint256 hash; @@ -982,38 +991,34 @@ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIV * banning/disconnecting us. We use this to determine which unaccepted * transactions from a whitelisted peer that we can safely relay. */ -static bool TxRelayMayResultInDisconnect(const CValidationState& state) -{ - assert(IsTransactionReason(state.GetReason())); - return state.GetReason() == ValidationInvalidReason::CONSENSUS; +static bool TxRelayMayResultInDisconnect(const TxValidationState& state) { + return state.GetResult() == TxValidationResult::TX_CONSENSUS; } /** - * Potentially ban a node based on the contents of a CValidationState object + * Potentially ban a node based on the contents of a BlockValidationState object * - * @param[in] via_compact_block: this bool is passed in because net_processing should + * @param[in] via_compact_block this bool is passed in because net_processing should * punish peers differently depending on whether the data was provided in a compact * block message or not. If the compact block had a valid header, but contained invalid * txs, the peer should not be punished. See BIP 152. * * @return Returns true if the peer was punished (probably disconnected) - * - * Changes here may need to be reflected in TxRelayMayResultInDisconnect(). */ -static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool via_compact_block, const std::string& message = "") { - switch (state.GetReason()) { - case ValidationInvalidReason::NONE: +static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, bool via_compact_block, const std::string& message = "") { + switch (state.GetResult()) { + case BlockValidationResult::BLOCK_RESULT_UNSET: break; // The node is providing invalid data: - case ValidationInvalidReason::CONSENSUS: - case ValidationInvalidReason::BLOCK_MUTATED: + case BlockValidationResult::BLOCK_CONSENSUS: + case BlockValidationResult::BLOCK_MUTATED: if (!via_compact_block) { LOCK(cs_main); Misbehaving(nodeid, 100, message); return true; } break; - case ValidationInvalidReason::CACHED_INVALID: + case BlockValidationResult::BLOCK_CACHED_INVALID: { LOCK(cs_main); CNodeState *node_state = State(nodeid); @@ -1029,30 +1034,24 @@ static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool v } break; } - case ValidationInvalidReason::BLOCK_INVALID_HEADER: - case ValidationInvalidReason::BLOCK_CHECKPOINT: - case ValidationInvalidReason::BLOCK_INVALID_PREV: + case BlockValidationResult::BLOCK_INVALID_HEADER: + case BlockValidationResult::BLOCK_CHECKPOINT: + case BlockValidationResult::BLOCK_INVALID_PREV: { LOCK(cs_main); Misbehaving(nodeid, 100, message); } return true; // Conflicting (but not necessarily invalid) data or different policy: - case ValidationInvalidReason::BLOCK_MISSING_PREV: + case BlockValidationResult::BLOCK_MISSING_PREV: { // TODO: Handle this much more gracefully (10 DoS points is super arbitrary) LOCK(cs_main); Misbehaving(nodeid, 10, message); } return true; - case ValidationInvalidReason::RECENT_CONSENSUS_CHANGE: - case ValidationInvalidReason::BLOCK_TIME_FUTURE: - case ValidationInvalidReason::TX_NOT_STANDARD: - case ValidationInvalidReason::TX_MISSING_INPUTS: - case ValidationInvalidReason::TX_PREMATURE_SPEND: - case ValidationInvalidReason::TX_WITNESS_MUTATED: - case ValidationInvalidReason::TX_CONFLICT: - case ValidationInvalidReason::TX_MEMPOOL_POLICY: + case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: + case BlockValidationResult::BLOCK_TIME_FUTURE: break; } if (message != "") { @@ -1061,6 +1060,39 @@ static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool v return false; } +/** + * Potentially ban a node based on the contents of a TxValidationState object + * + * @return Returns true if the peer was punished (probably disconnected) + * + * Changes here may need to be reflected in TxRelayMayResultInDisconnect(). + */ +static bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = "") { + switch (state.GetResult()) { + case TxValidationResult::TX_RESULT_UNSET: + break; + // The node is providing invalid data: + case TxValidationResult::TX_CONSENSUS: + { + LOCK(cs_main); + Misbehaving(nodeid, 100, message); + return true; + } + // Conflicting (but not necessarily invalid) data or different policy: + case TxValidationResult::TX_RECENT_CONSENSUS_CHANGE: + case TxValidationResult::TX_NOT_STANDARD: + case TxValidationResult::TX_MISSING_INPUTS: + case TxValidationResult::TX_PREMATURE_SPEND: + case TxValidationResult::TX_WITNESS_MUTATED: + case TxValidationResult::TX_CONFLICT: + case TxValidationResult::TX_MEMPOOL_POLICY: + break; + } + if (message != "") { + LogPrint(BCLog::NET, "peer=%d: %s\n", nodeid, message); + } + return false; +} @@ -1092,6 +1124,16 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + // Blocks don't typically have more than 4000 transactions, so this should + // be at least six blocks (~1 hr) worth of transactions that we can store. + // If the number of transactions appearing in a block goes up, or if we are + // seeing getdata requests more than an hour after initial announcement, we + // can increase this number. + // The false positive rate of 1/1M should come out to less than 1 + // transaction per day that would be inadvertently ignored (which is the + // same probability that we have in the reject filter). + g_recent_confirmed_transactions.reset(new CRollingBloomFilter(24000, 0.000001)); + const Consensus::Params& consensusParams = Params().GetConsensus(); // Stale tip checking and peer eviction are on two different timers, but we // don't want them to get out of sync due to drift in the scheduler, so we @@ -1105,40 +1147,63 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS * Evict orphan txn pool entries (EraseOrphanTx) based on a newly connected * block. Also save the time of the last tip update. */ -void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { - LOCK(g_cs_orphans); +void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) +{ + { + LOCK(g_cs_orphans); - std::vector<uint256> vOrphanErase; + std::vector<uint256> vOrphanErase; - for (const CTransactionRef& ptx : pblock->vtx) { - const CTransaction& tx = *ptx; + for (const CTransactionRef& ptx : pblock->vtx) { + const CTransaction& tx = *ptx; - // Which orphan pool entries must we evict? - for (const auto& txin : tx.vin) { - auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = *(*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); + // Which orphan pool entries must we evict? + for (const auto& txin : tx.vin) { + auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = *(*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } } } - } - // Erase orphan transactions included or precluded by this block - if (vOrphanErase.size()) { - int nErased = 0; - for (const uint256& orphanHash : vOrphanErase) { - nErased += EraseOrphanTx(orphanHash); + // Erase orphan transactions included or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + for (const uint256& orphanHash : vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); + } + + g_last_tip_update = GetTime(); + } + { + LOCK(g_cs_recent_confirmed_transactions); + for (const auto& ptx : pblock->vtx) { + g_recent_confirmed_transactions->insert(ptx->GetHash()); } - LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } +} - g_last_tip_update = GetTime(); +void PeerLogicValidation::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) +{ + // To avoid relay problems with transactions that were previously + // confirmed, clear our filter of recently confirmed transactions whenever + // there's a reorg. + // This means that in a 1-block reorg (where 1 block is disconnected and + // then another block reconnected), our filter will drop to having only one + // block's worth of transactions in it, but that should be fine, since + // presumably the most common case of relaying a confirmed transaction + // should be just after a new block containing it is found. + LOCK(g_cs_recent_confirmed_transactions); + g_recent_confirmed_transactions->reset(); } // All of the following cache a recent block, and are protected by cs_most_recent_block -static CCriticalSection cs_most_recent_block; +static RecursiveMutex cs_most_recent_block; static std::shared_ptr<const CBlock> most_recent_block GUARDED_BY(cs_most_recent_block); static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block GUARDED_BY(cs_most_recent_block); static uint256 most_recent_block_hash GUARDED_BY(cs_most_recent_block); @@ -1229,7 +1294,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB * Handle invalid block rejection and consequent peer banning, maintain which * peers announce compact blocks. */ -void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationState& state) { +void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidationState& state) { LOCK(cs_main); const uint256 hash(block.GetHash()); @@ -1240,7 +1305,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationSta if (state.IsInvalid() && it != mapBlockSource.end() && State(it->second.first)) { - MaybePunishNode(/*nodeid=*/ it->second.first, state, /*via_compact_block=*/ !it->second.second); + MaybePunishNodeForBlock(/*nodeid=*/ it->second.first, state, /*via_compact_block=*/ !it->second.second); } // Check that: // 1. The block is valid @@ -1287,12 +1352,14 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) LOCK(g_cs_orphans); if (mapOrphanTransactions.count(inv.hash)) return true; } - const CCoinsViewCache& coins_cache = ::ChainstateActive().CoinsTip(); + + { + LOCK(g_cs_recent_confirmed_transactions); + if (g_recent_confirmed_transactions->contains(inv.hash)) return true; + } return recentRejects->contains(inv.hash) || - mempool.exists(inv.hash) || - coins_cache.HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 - coins_cache.HaveCoinInCache(COutPoint(inv.hash, 1)); + mempool.exists(inv.hash); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -1317,7 +1384,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma // Relay to a limited number of other nodes // Use deterministic randomness to send to the same nodes for 24 hours - // at a time so the addrKnowns of the chosen nodes prevent repeats + // at a time so the m_addr_knowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); FastRandomContext insecure_rand; @@ -1378,7 +1445,7 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c } } // release cs_main before calling ActivateBestChain if (need_activate_chain) { - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, Params(), a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", FormatStateMessage(state)); } @@ -1674,11 +1741,10 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve } } - CValidationState state; - CBlockHeader first_invalid_header; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { + BlockValidationState state; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { if (state.IsInvalid()) { - MaybePunishNode(pfrom->GetId(), state, via_compact_block, "invalid header received"); + MaybePunishNodeForBlock(pfrom->GetId(), state, via_compact_block, "invalid header received"); return false; } } @@ -1814,14 +1880,13 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se const CTransactionRef porphanTx = orphan_it->second.tx; const CTransaction& orphanTx = *porphanTx; NodeId fromPeer = orphan_it->second.fromPeer; - bool fMissingInputs2 = false; - // Use a new CValidationState because orphans come from different peers (and we call - // MaybePunishNode based on the source peer from the orphan map, not based on the peer + // Use a new TxValidationState because orphans come from different peers (and we call + // MaybePunishNodeForTx based on the source peer from the orphan map, not based on the peer // that relayed the previous transaction). - CValidationState orphan_state; + TxValidationState orphan_state; if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &fMissingInputs2, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanHash, *connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { @@ -1834,10 +1899,10 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se } EraseOrphanTx(orphanHash); done = true; - } else if (!fMissingInputs2) { + } else if (orphan_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { if (orphan_state.IsInvalid()) { // Punish peer that gave us an invalid orphan tx - if (MaybePunishNode(fromPeer, orphan_state, /*via_compact_block*/ false)) { + if (MaybePunishNodeForTx(fromPeer, orphan_state)) { setMisbehaving.insert(fromPeer); } LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); @@ -1845,8 +1910,7 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se // Has inputs but not accepted to mempool // Probably non-standard or insufficient fee LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); - assert(IsTransactionReason(orphan_state.GetReason())); - if (!orphanTx.HasWitness() && orphan_state.GetReason() != ValidationInvalidReason::TX_WITNESS_MUTATED) { + if (!orphanTx.HasWitness() && orphan_state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { // Do not use rejection cache for witness transactions or // witness-stripped transactions, as they can have been malleated. // See https://github.com/bitcoin/bitcoin/issues/8279 for details. @@ -1860,7 +1924,7 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se } } -bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc) +bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -2136,7 +2200,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); - if (g_banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them + if (banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { @@ -2291,7 +2355,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_most_recent_block); a_recent_block = most_recent_block; } - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, Params(), a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", FormatStateMessage(state)); } @@ -2471,8 +2535,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK2(cs_main, g_cs_orphans); - bool fMissingInputs = false; - CValidationState state; + TxValidationState state; CNodeState* nodestate = State(pfrom->GetId()); nodestate->m_tx_download.m_tx_announced.erase(inv.hash); @@ -2482,7 +2545,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::list<CTransactionRef> lRemovedTxn; if (!AlreadyHave(inv) && - AcceptToMemoryPool(mempool, state, ptx, &fMissingInputs, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { mempool.check(&::ChainstateActive().CoinsTip()); RelayTransaction(tx.GetHash(), *connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { @@ -2504,7 +2567,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Recursively process any orphan transactions that depended on this one ProcessOrphanTx(connman, pfrom->orphan_work_set, lRemovedTxn); } - else if (fMissingInputs) + else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected for (const CTxIn& txin : tx.vin) { @@ -2537,8 +2600,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(tx.GetHash()); } } else { - assert(IsTransactionReason(state.GetReason())); - if (!tx.HasWitness() && state.GetReason() != ValidationInvalidReason::TX_WITNESS_MUTATED) { + if (!tx.HasWitness() && state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { // Do not use rejection cache for witness transactions or // witness-stripped transactions, as they can have been malleated. // See https://github.com/bitcoin/bitcoin/issues/8279 for details. @@ -2593,7 +2655,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); - MaybePunishNode(pfrom->GetId(), state, /*via_compact_block*/ false); + MaybePunishNodeForTx(pfrom->GetId(), state); } return true; } @@ -2627,10 +2689,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } const CBlockIndex *pindex = nullptr; - CValidationState state; + BlockValidationState state; if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { if (state.IsInvalid()) { - MaybePunishNode(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); + MaybePunishNodeForBlock(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return true; } } @@ -2772,7 +2834,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, banman, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be @@ -2990,7 +3052,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::vector<CAddress> vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { - if (!g_banman->IsBanned(addr)) { + if (!banman->IsBanned(addr)) { pfrom->PushAddress(addr, insecure_rand); } } @@ -3272,41 +3334,37 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return false; // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); - pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; + pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman->GetReceiveFloodSize(); fMoreWork = !pfrom->vProcessMsg.empty(); } CNetMessage& msg(msgs.front()); msg.SetVersion(pfrom->GetRecvVersion()); - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrint(BCLog::NET, "PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + // Check network magic + if (!msg.m_valid_netmagic) { + LogPrint(BCLog::NET, "PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); pfrom->fDisconnect = true; return false; } - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) + // Check header + if (!msg.m_valid_header) { - LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); return fMoreWork; } - std::string strCommand = hdr.GetCommand(); + const std::string& strCommand = msg.m_command; // Message size - unsigned int nMessageSize = hdr.nMessageSize; + unsigned int nMessageSize = msg.m_message_size; // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + CDataStream& vRecv = msg.m_recv; + if (!msg.m_valid_checksum) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR peer=%d\n", __func__, + SanitizeString(strCommand), nMessageSize, pfrom->GetId()); return fMoreWork; } @@ -3314,37 +3372,15 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.m_time, chainparams, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) fMoreWork = true; - } - catch (const std::ios_base::failure& e) - { - if (strstr(e.what(), "end of data")) { - // Allow exceptions from under-length message on vRecv - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "size too large")) { - // Allow exceptions from over-long size - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { - // Allow exceptions from non-canonical encoding - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "Superfluous witness record")) { - // Allow exceptions from illegal witness encoding - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "Unknown transaction optional data")) { - // Allow exceptions from unknown witness encoding - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else { - PrintExceptionContinue(&e, "ProcessMessages()"); - } - } - catch (const std::exception& e) { - PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (const std::exception& e) { + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what(), typeid(e).name()); } catch (...) { - PrintExceptionContinue(nullptr, "ProcessMessages()"); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(strCommand), nMessageSize); } if (!fRet) { @@ -3561,6 +3597,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Address refresh broadcast int64_t nNow = GetTimeMicros(); + auto current_time = GetTime<std::chrono::microseconds>(); + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { AdvertiseLocal(pto); pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); @@ -3573,11 +3611,12 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); + assert(pto->m_addr_known); for (const CAddress& addr : pto->vAddrToSend) { - if (!pto->addrKnown.contains(addr.GetKey())) + if (!pto->m_addr_known->contains(addr.GetKey())) { - pto->addrKnown.insert(addr.GetKey()); + pto->m_addr_known->insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) @@ -3781,13 +3820,13 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(pto->m_tx_relay->cs_tx_inventory); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(PF_NOBAN); - if (pto->m_tx_relay->nNextInvSend < nNow) { + if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; if (pto->fInbound) { - pto->m_tx_relay->nNextInvSend = connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL); + pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; } else { // Use half the delay for outbound peers, as there is less privacy concern for them. - pto->m_tx_relay->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> 1); + pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, std::chrono::seconds{INVENTORY_BROADCAST_INTERVAL >> 1}); } } @@ -3902,7 +3941,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling - const auto current_time = GetTime<std::chrono::microseconds>(); + current_time = GetTime<std::chrono::microseconds>(); // nNow is the current system time (GetTimeMicros is not mockable) and // should be replaced by the mockable current_time eventually nNow = GetTimeMicros(); diff --git a/src/net_processing.h b/src/net_processing.h index e8bc3580dd..6f26abc209 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,7 @@ #include <consensus/params.h> #include <sync.h> -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; @@ -33,6 +33,7 @@ public: * Overridden from CValidationInterface. */ void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override; /** * Overridden from CValidationInterface. */ @@ -40,7 +41,7 @@ public: /** * Overridden from CValidationInterface. */ - void BlockChecked(const CBlock& block, const CValidationState& state) override; + void BlockChecked(const CBlock& block, const BlockValidationState& state) override; /** * Overridden from CValidationInterface. */ diff --git a/src/net_types.h b/src/net_types.h new file mode 100644 index 0000000000..d55a8cde6c --- /dev/null +++ b/src/net_types.h @@ -0,0 +1,15 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NET_TYPES_H +#define BITCOIN_NET_TYPES_H + +#include <map> + +class CBanEntry; +class CSubNet; + +using banmap_t = std::map<CSubNet, CBanEntry>; + +#endif // BITCOIN_NET_TYPES_H diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 4fbfa2b5c8..ce3e17197e 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netaddress.h> #include <hash.h> #include <util/strencodings.h> +#include <util/asmap.h> #include <tinyformat.h> static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; @@ -400,6 +401,39 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } +uint32_t CNetAddr::GetNetClass() const { + uint32_t net_class = NET_IPV6; + if (IsLocal()) { + net_class = 255; + } + if (IsInternal()) { + net_class = NET_INTERNAL; + } else if (!IsRoutable()) { + net_class = NET_UNROUTABLE; + } else if (IsIPv4() || IsRFC6145() || IsRFC6052() || IsRFC3964() || IsRFC4380()) { + net_class = NET_IPV4; + } else if (IsTor()) { + net_class = NET_ONION; + } + return net_class; +} + +uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { + uint32_t net_class = GetNetClass(); + if (asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { + return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. + } + std::vector<bool> ip_bits(128); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + uint32_t mapped_as = Interpret(asmap, ip_bits); + return mapped_as; +} + /** * Get the canonical identifier of our network group * @@ -410,56 +444,61 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const * @note No two connections will be attempted to addresses with the same network * group. */ -std::vector<unsigned char> CNetAddr::GetGroup() const +std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) const { std::vector<unsigned char> vchRet; - int nClass = NET_IPV6; + uint32_t net_class = GetNetClass(); + // If non-empty asmap is supplied and the address is IPv4/IPv6, + // return ASN to be used for bucketing. + uint32_t asn = GetMappedAS(asmap); + if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). + vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + + vchRet.push_back(net_class); int nStartByte = 0; int nBits = 16; // all local addresses belong to the same group if (IsLocal()) { - nClass = 255; nBits = 0; } // all internal-usage addresses get their own group if (IsInternal()) { - nClass = NET_INTERNAL; nStartByte = sizeof(g_internal_prefix); nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; } // all other unroutable addresses belong to the same group else if (!IsRoutable()) { - nClass = NET_UNROUTABLE; nBits = 0; } // for IPv4 addresses, '1' + the 16 higher-order bits of the IP // includes mapped IPv4, SIIT translated IPv4, and the well-known prefix else if (IsIPv4() || IsRFC6145() || IsRFC6052()) { - nClass = NET_IPV4; nStartByte = 12; } // for 6to4 tunnelled addresses, use the encapsulated IPv4 address else if (IsRFC3964()) { - nClass = NET_IPV4; nStartByte = 2; } // for Teredo-tunnelled IPv6 addresses, use the encapsulated IPv4 address else if (IsRFC4380()) { - vchRet.push_back(NET_IPV4); vchRet.push_back(GetByte(3) ^ 0xFF); vchRet.push_back(GetByte(2) ^ 0xFF); return vchRet; } else if (IsTor()) { - nClass = NET_ONION; nStartByte = 6; nBits = 4; } @@ -470,8 +509,6 @@ std::vector<unsigned char> CNetAddr::GetGroup() const else nBits = 32; - vchRet.push_back(nClass); - // push our ip onto vchRet byte by byte... while (nBits >= 8) { diff --git a/src/netaddress.h b/src/netaddress.h index fbb1553338..078234595c 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -54,7 +54,7 @@ class CNetAddr bool IsIPv4() const; // IPv4 mapped address (::FFFF:0:0/96, 0.0.0.0/0) bool IsIPv6() const; // IPv6 address (not mapped IPv4, not Tor) bool IsRFC1918() const; // IPv4 private networks (10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12) - bool IsRFC2544() const; // IPv4 inter-network communications (192.18.0.0/15) + bool IsRFC2544() const; // IPv4 inter-network communications (198.18.0.0/15) bool IsRFC6598() const; // IPv4 ISP-level NAT (100.64.0.0/10) bool IsRFC5737() const; // IPv4 documentation addresses (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) bool IsRFC3849() const; // IPv6 documentation address (2001:0DB8::/32) @@ -78,7 +78,14 @@ class CNetAddr unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; - std::vector<unsigned char> GetGroup() const; + uint32_t GetNetClass() const; + + // The AS on the BGP path to the node we use to diversify + // peers in AddrMan bucketing based on the AS infrastructure. + // The ip->AS mapping depends on how asmap is constructed. + uint32_t GetMappedAS(const std::vector<bool> &asmap) const; + + std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); diff --git a/src/netbase.cpp b/src/netbase.cpp index 0148aea428..a70179cb16 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,9 @@ #include <sync.h> #include <tinyformat.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/string.h> +#include <util/system.h> #include <atomic> @@ -27,7 +28,7 @@ #endif // Settings -static CCriticalSection cs_proxyInfos; +static RecursiveMutex cs_proxyInfos; static proxyType proxyInfo[NET_MAX] GUARDED_BY(cs_proxyInfos); static proxyType nameProxy GUARDED_BY(cs_proxyInfos); int nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -59,10 +60,14 @@ std::string GetNetworkName(enum Network net) { } } -bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) +bool static LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { vIP.clear(); + if (!ValidAsCString(name)) { + return false; + } + { CNetAddr addr; // From our perspective, onion addresses are not hostnames but rather @@ -71,7 +76,7 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign // getaddrinfo to decode them and it wouldn't make sense to resolve // them, we return a network address representing it instead. See // CNetAddr::SetSpecial(const std::string&) for more details. - if (addr.SetSpecial(std::string(pszName))) { + if (addr.SetSpecial(name)) { vIP.push_back(addr); return true; } @@ -93,7 +98,7 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign // hostname lookups. aiHint.ai_flags = fAllowLookup ? AI_ADDRCONFIG : AI_NUMERICHOST; struct addrinfo *aiRes = nullptr; - int nErr = getaddrinfo(pszName, nullptr, &aiHint, &aiRes); + int nErr = getaddrinfo(name.c_str(), nullptr, &aiHint, &aiRes); if (nErr) return false; @@ -131,7 +136,7 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign /** * Resolve a host string to its corresponding network addresses. * - * @param pszName The string representing a host. Could be a name or a numerical + * @param name The string representing a host. Could be a name or a numerical * IP address (IPv6 addresses in their bracketed form are * allowed). * @param[out] vIP The resulting network addresses to which the specified host @@ -143,28 +148,34 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign * @see Lookup(const char *, std::vector<CService>&, int, bool, unsigned int) * for additional parameter descriptions. */ -bool LookupHost(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) +bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { - std::string strHost(pszName); + if (!ValidAsCString(name)) { + return false; + } + std::string strHost = name; if (strHost.empty()) return false; if (strHost.front() == '[' && strHost.back() == ']') { strHost = strHost.substr(1, strHost.size() - 2); } - return LookupIntern(strHost.c_str(), vIP, nMaxSolutions, fAllowLookup); + return LookupIntern(strHost, vIP, nMaxSolutions, fAllowLookup); } /** * Resolve a host string to its first corresponding network address. * - * @see LookupHost(const char *, std::vector<CNetAddr>&, unsigned int, bool) for + * @see LookupHost(const std::string&, std::vector<CNetAddr>&, unsigned int, bool) for * additional parameter descriptions. */ -bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup) +bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup) { + if (!ValidAsCString(name)) { + return false; + } std::vector<CNetAddr> vIP; - LookupHost(pszName, vIP, 1, fAllowLookup); + LookupHost(name, vIP, 1, fAllowLookup); if(vIP.empty()) return false; addr = vIP.front(); @@ -174,7 +185,7 @@ bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup) /** * Resolve a service string to its corresponding service. * - * @param pszName The string representing a service. Could be a name or a + * @param name The string representing a service. Could be a name or a * numerical IP address (IPv6 addresses should be in their * disambiguated bracketed form), optionally followed by a port * number. (e.g. example.com:8333 or @@ -191,16 +202,17 @@ bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup) * @returns Whether or not the service string successfully resolved to any * resulting services. */ -bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions) +bool Lookup(const std::string& name, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions) { - if (pszName[0] == 0) + if (name.empty() || !ValidAsCString(name)) { return false; + } int port = portDefault; std::string hostname; - SplitHostPort(std::string(pszName), port, hostname); + SplitHostPort(name, port, hostname); std::vector<CNetAddr> vIP; - bool fRet = LookupIntern(hostname.c_str(), vIP, nMaxSolutions, fAllowLookup); + bool fRet = LookupIntern(hostname, vIP, nMaxSolutions, fAllowLookup); if (!fRet) return false; vAddr.resize(vIP.size()); @@ -215,10 +227,13 @@ bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, * @see Lookup(const char *, std::vector<CService>&, int, bool, unsigned int) * for additional parameter descriptions. */ -bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLookup) +bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllowLookup) { + if (!ValidAsCString(name)) { + return false; + } std::vector<CService> vService; - bool fRet = Lookup(pszName, vService, portDefault, fAllowLookup, 1); + bool fRet = Lookup(name, vService, portDefault, fAllowLookup, 1); if (!fRet) return false; addr = vService[0]; @@ -235,12 +250,15 @@ bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLoo * @see Lookup(const char *, CService&, int, bool) for additional parameter * descriptions. */ -CService LookupNumeric(const char *pszName, int portDefault) +CService LookupNumeric(const std::string& name, int portDefault) { + if (!ValidAsCString(name)) { + return {}; + } CService addr; // "1.2:345" will fail to resolve the ip, but will still set the port. // If the ip fails to resolve, re-init the result. - if(!Lookup(pszName, addr, portDefault, false)) + if(!Lookup(name, addr, portDefault, false)) addr = CService(); return addr; } @@ -605,7 +623,7 @@ static void LogConnectFailure(bool manual_connection, const char* fmt, const Arg * @param nTimeout Wait this many milliseconds for the connection to be * established. * @param manual_connection Whether or not the connection was manually requested - * (e.g. thru the addnode RPC) + * (e.g. through the addnode RPC) * * @returns Whether or not a connection was successfully made. */ @@ -709,7 +727,7 @@ bool GetProxy(enum Network net, proxyType &proxyInfoOut) { /** * Set the name proxy to use for all connections to nodes specified by a - * hostname. After setting this proxy, connecting to a node sepcified by a + * hostname. After setting this proxy, connecting to a node specified by a * hostname won't result in a local lookup of said hostname, rather, connect to * the node by asking the name proxy for a proxy connection to the hostname, * effectively delegating the hostname lookup to the specified proxy. @@ -763,17 +781,16 @@ bool IsProxy(const CNetAddr &addr) { * @param hSocket The socket on which to connect to the SOCKS5 proxy. * @param nTimeout Wait this many milliseconds for the connection to the SOCKS5 * proxy to be established. - * @param outProxyConnectionFailed[out] Whether or not the connection to the + * @param[out] outProxyConnectionFailed Whether or not the connection to the * SOCKS5 proxy failed. * * @returns Whether or not the operation succeeded. */ -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool *outProxyConnectionFailed) +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool& outProxyConnectionFailed) { // first connect to proxy server if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout, true)) { - if (outProxyConnectionFailed) - *outProxyConnectionFailed = true; + outProxyConnectionFailed = true; return false; } // do socks negotiation @@ -796,23 +813,25 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int * Parse and resolve a specified subnet string into the appropriate internal * representation. * - * @param pszName A string representation of a subnet of the form `network + * @param strSubnet A string representation of a subnet of the form `network * address [ "/", ( CIDR-style suffix | netmask ) ]`(e.g. * `2001:db8::/32`, `192.0.2.0/255.255.255.0`, or `8.8.8.8`). * @param ret The resulting internal representation of a subnet. * * @returns Whether the operation succeeded or not. */ -bool LookupSubNet(const char* pszName, CSubNet& ret) +bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) { - std::string strSubnet(pszName); + if (!ValidAsCString(strSubnet)) { + return false; + } size_t slash = strSubnet.find_last_of('/'); std::vector<CNetAddr> vIP; std::string strAddress = strSubnet.substr(0, slash); - // TODO: Use LookupHost(const char *, CNetAddr&, bool) instead to just get + // TODO: Use LookupHost(const std::string&, CNetAddr&, bool) instead to just get // one CNetAddr. - if (LookupHost(strAddress.c_str(), vIP, 1, false)) + if (LookupHost(strAddress, vIP, 1, false)) { CNetAddr network = vIP[0]; if (slash != strSubnet.npos) @@ -827,7 +846,7 @@ bool LookupSubNet(const char* pszName, CSubNet& ret) else // If not a valid number, try full netmask syntax { // Never allow lookup for netmask - if (LookupHost(strNetmask.c_str(), vIP, 1, false)) { + if (LookupHost(strNetmask, vIP, 1, false)) { ret = CSubNet(network, vIP[0]); return ret.IsValid(); } diff --git a/src/netbase.h b/src/netbase.h index 313a575687..ac4cd97673 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,15 +45,15 @@ bool IsProxy(const CNetAddr &addr); bool SetNameProxy(const proxyType &addrProxy); bool HaveNameProxy(); bool GetNameProxy(proxyType &nameProxyOut); -bool LookupHost(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup); -bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup); -bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLookup); -bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); -CService LookupNumeric(const char *pszName, int portDefault = 0); -bool LookupSubNet(const char *pszName, CSubNet& subnet); +bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup); +bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup); +bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllowLookup); +bool Lookup(const std::string& name, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); +CService LookupNumeric(const std::string& name, int portDefault = 0); +bool LookupSubNet(const std::string& strSubnet, CSubNet& subnet); SOCKET CreateSocket(const CService &addrConnect); bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocketRet, int nTimeout, bool manual_connection); -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed); +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool& outProxyConnectionFailed); /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); /** Close socket and set hSocket to INVALID_SOCKET */ diff --git a/src/node/coin.cpp b/src/node/coin.cpp index ad8d1d3af4..f4f86cdbe9 100644 --- a/src/node/coin.cpp +++ b/src/node/coin.cpp @@ -4,14 +4,16 @@ #include <node/coin.h> +#include <node/context.h> #include <txmempool.h> #include <validation.h> -void FindCoins(std::map<COutPoint, Coin>& coins) +void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins) { - LOCK2(cs_main, ::mempool.cs); + assert(node.mempool); + LOCK2(cs_main, node.mempool->cs); CCoinsViewCache& chain_view = ::ChainstateActive().CoinsTip(); - CCoinsViewMemPool mempool_view(&chain_view, ::mempool); + CCoinsViewMemPool mempool_view(&chain_view, *node.mempool); for (auto& coin : coins) { if (!mempool_view.GetCoin(coin.first, coin.second)) { // Either the coin is not in the CCoinsViewCache or is spent. Clear it. diff --git a/src/node/coin.h b/src/node/coin.h index eb95b75cfb..908850e2a5 100644 --- a/src/node/coin.h +++ b/src/node/coin.h @@ -9,14 +9,16 @@ class COutPoint; class Coin; +struct NodeContext; /** * Look up unspent output information. Returns coins in the mempool and in the * current chain UTXO set. Iterates through all the keys in the map and * populates the values. * + * @param[in] node The node context to use for lookup * @param[in,out] coins map to fill */ -void FindCoins(std::map<COutPoint, Coin>& coins); +void FindCoins(const NodeContext& node, std::map<COutPoint, Coin>& coins); #endif // BITCOIN_NODE_COIN_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 57fa158ad2..a818f06d51 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -14,9 +14,6 @@ #include <map> -#include <boost/thread.hpp> - - static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); @@ -38,6 +35,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); @@ -51,7 +49,6 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { @@ -61,6 +58,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c11aab8bd..a19af0fd1b 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,16 +15,17 @@ class CCoinsView; struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + CAmount nTotalAmount{0}; + + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set diff --git a/src/node/context.cpp b/src/node/context.cpp new file mode 100644 index 0000000000..26a01420c8 --- /dev/null +++ b/src/node/context.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/context.h> + +#include <banman.h> +#include <interfaces/chain.h> +#include <net.h> +#include <net_processing.h> + +NodeContext::NodeContext() {} +NodeContext::~NodeContext() {} diff --git a/src/node/context.h b/src/node/context.h new file mode 100644 index 0000000000..dab5b5d048 --- /dev/null +++ b/src/node/context.h @@ -0,0 +1,45 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_CONTEXT_H +#define BITCOIN_NODE_CONTEXT_H + +#include <memory> +#include <vector> + +class BanMan; +class CConnman; +class CTxMemPool; +class PeerLogicValidation; +namespace interfaces { +class Chain; +class ChainClient; +} // namespace interfaces + +//! NodeContext struct containing references to chain state and connection +//! state. +//! +//! This is used by init, rpc, and test code to pass object references around +//! without needing to declare the same variables and parameters repeatedly, or +//! to use globals. More variables could be added to this struct (particularly +//! references to validation objects) to eliminate use of globals +//! and make code more modular and testable. The struct isn't intended to have +//! any member functions. It should just be a collection of references that can +//! be used without pulling in unwanted dependencies or functionality. +struct NodeContext { + std::unique_ptr<CConnman> connman; + CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct + std::unique_ptr<PeerLogicValidation> peer_logic; + std::unique_ptr<BanMan> banman; + std::unique_ptr<interfaces::Chain> chain; + std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; + + //! Declare default constructor and destructor that are not inline, so code + //! instantiating the NodeContext struct doesn't need to #include class + //! definitions for all the unique_ptr members. + NodeContext(); + ~NodeContext(); +}; + +#endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 12559c5a5f..8678b33cf3 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -1,12 +1,14 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <amount.h> #include <coins.h> #include <consensus/tx_verify.h> #include <node/psbt.h> #include <policy/policy.h> #include <policy/settings.h> +#include <tinyformat.h> #include <numeric> @@ -30,15 +32,28 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) // Check for a UTXO CTxOut utxo; if (psbtx.GetInputUTXO(utxo, i)) { + if (!MoneyRange(utxo.nValue) || !MoneyRange(in_amt + utxo.nValue)) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u has invalid value", i)); + return result; + } in_amt += utxo.nValue; input_analysis.has_utxo = true; } else { + if (input.non_witness_utxo && psbtx.tx->vin[i].prevout.n >= input.non_witness_utxo->vout.size()) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u specifies invalid prevout", i)); + return result; + } input_analysis.has_utxo = false; input_analysis.is_final = false; input_analysis.next = PSBTRole::UPDATER; calc_fee = false; } + if (!utxo.IsNull() && utxo.scriptPubKey.IsUnspendable()) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u spends unspendable output", i)); + return result; + } + // Check if it is final if (!utxo.IsNull() && !PSBTInputSigned(input)) { input_analysis.is_final = false; @@ -79,9 +94,16 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) // Get the output amount CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), [](CAmount a, const CTxOut& b) { + if (!MoneyRange(a) || !MoneyRange(b.nValue) || !MoneyRange(a + b.nValue)) { + return CAmount(-1); + } return a += b.nValue; } ); + if (!MoneyRange(out_amt)) { + result.SetInvalid(strprintf("PSBT is not valid. Output amount invalid")); + return result; + } // Get the fee CAmount fee = in_amt - out_amt; diff --git a/src/node/psbt.h b/src/node/psbt.h index e04366a20f..7384dc415c 100644 --- a/src/node/psbt.h +++ b/src/node/psbt.h @@ -30,6 +30,17 @@ struct PSBTAnalysis { Optional<CAmount> fee; //!< Amount of fee being paid by the transaction std::vector<PSBTInputAnalysis> inputs; //!< More information about the individual inputs of the transaction PSBTRole next; //!< Which of the BIP 174 roles needs to handle the transaction next + std::string error; //!< Error message + + void SetInvalid(std::string err_msg) + { + estimated_vsize = nullopt; + estimated_feerate = nullopt; + fee = nullopt; + inputs.clear(); + next = PSBTRole::CREATOR; + error = err_msg; + } }; /** diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 7783671a6c..1bb9b88d00 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/validation.h> #include <net.h> #include <net_processing.h> +#include <node/context.h> #include <util/validation.h> #include <validation.h> #include <validationinterface.h> @@ -13,12 +14,13 @@ #include <future> -TransactionError BroadcastTransaction(const CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback) +TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback) { // BroadcastTransaction can be called by either sendrawtransaction RPC or wallet RPCs. - // g_connman is assigned both before chain clients and before RPC server is accepting calls, - // and reset after chain clients and RPC sever are stopped. g_connman should never be null here. - assert(g_connman); + // node.connman is assigned both before chain clients and before RPC server is accepting calls, + // and reset after chain clients and RPC sever are stopped. node.connman should never be null here. + assert(node.connman); + assert(node.mempool); std::promise<void> promise; uint256 hashTx = tx->GetHash(); bool callback_set = false; @@ -30,24 +32,22 @@ TransactionError BroadcastTransaction(const CTransactionRef tx, std::string& err CCoinsViewCache &view = ::ChainstateActive().CoinsTip(); for (size_t o = 0; o < tx->vout.size(); o++) { const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); - // IsSpent doesnt mean the coin is spent, it means the output doesnt' exist. + // IsSpent doesn't mean the coin is spent, it means the output doesn't exist. // So if the output does exist, then this transaction exists in the chain. if (!existingCoin.IsSpent()) return TransactionError::ALREADY_IN_CHAIN; } - if (!mempool.exists(hashTx)) { + if (!node.mempool->exists(hashTx)) { // Transaction is not already in the mempool. Submit it. - CValidationState state; - bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, std::move(tx), &fMissingInputs, + TxValidationState state; + if (!AcceptToMemoryPool(*node.mempool, state, std::move(tx), nullptr /* plTxnReplaced */, false /* bypass_limits */, max_tx_fee)) { + err_string = FormatStateMessage(state); if (state.IsInvalid()) { - err_string = FormatStateMessage(state); - return TransactionError::MEMPOOL_REJECTED; - } else { - if (fMissingInputs) { + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { return TransactionError::MISSING_INPUTS; } - err_string = FormatStateMessage(state); + return TransactionError::MEMPOOL_REJECTED; + } else { return TransactionError::MEMPOOL_ERROR; } } @@ -79,7 +79,7 @@ TransactionError BroadcastTransaction(const CTransactionRef tx, std::string& err } if (relay) { - RelayTransaction(hashTx, *g_connman); + RelayTransaction(hashTx, *node.connman); } return TransactionError::OK; diff --git a/src/node/transaction.h b/src/node/transaction.h index a3e56544a7..a85dfb8ace 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -9,6 +9,8 @@ #include <primitives/transaction.h> #include <util/error.h> +struct NodeContext; + /** * Submit a transaction to the mempool and (optionally) relay it to all P2P peers. * @@ -18,13 +20,14 @@ * NOT be set while cs_main, cs_mempool or cs_wallet are held to avoid * deadlock. * + * @param[in] node reference to node context * @param[in] tx the transaction to broadcast - * @param[out] &err_string reference to std::string to fill with error string if available + * @param[out] err_string reference to std::string to fill with error string if available * @param[in] max_tx_fee reject txs with fees higher than this (if 0, accept any fee) * @param[in] relay flag if both mempool insertion and p2p relay are requested - * @param[in] wait_callback, wait until callbacks have been processed to avoid stale result due to a sequentially RPC. + * @param[in] wait_callback wait until callbacks have been processed to avoid stale result due to a sequentially RPC. * return error */ -NODISCARD TransactionError BroadcastTransaction(CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); +NODISCARD TransactionError BroadcastTransaction(NodeContext& node, CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 0000000000..702a0cbe53 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,50 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include <uint256.h> +#include <serialize.h> + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } + +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/noui.cpp b/src/noui.cpp index 14d6183d24..11cfe7f94d 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,7 +45,7 @@ bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& ca if (!fSecure) { LogPrintf("%s%s\n", strCaption, message); } - tfm::format(std::cerr, "%s%s\n", strCaption.c_str(), message.c_str()); + tfm::format(std::cerr, "%s%s\n", strCaption, message); return false; } @@ -66,28 +66,31 @@ void noui_connect() noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage); } -bool noui_ThreadSafeMessageBoxSuppressed(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -bool noui_ThreadSafeQuestionSuppressed(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -void noui_InitMessageSuppressed(const std::string& message) +void noui_InitMessageRedirect(const std::string& message) { + LogPrintf("init message: %s\n", message); } -void noui_suppress() +void noui_test_redirect() { noui_ThreadSafeMessageBoxConn.disconnect(); noui_ThreadSafeQuestionConn.disconnect(); noui_InitMessageConn.disconnect(); - noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxSuppressed); - noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionSuppressed); - noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageSuppressed); + noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxRedirect); + noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionRedirect); + noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageRedirect); } void noui_reconnect() @@ -96,4 +99,4 @@ void noui_reconnect() noui_ThreadSafeQuestionConn.disconnect(); noui_InitMessageConn.disconnect(); noui_connect(); -}
\ No newline at end of file +} diff --git a/src/noui.h b/src/noui.h index 854aeeacca..5e5767b453 100644 --- a/src/noui.h +++ b/src/noui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2018 The Bitcoin Core developers +// Copyright (c) 2013-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,10 +17,10 @@ void noui_InitMessage(const std::string& message); /** Connect all bitcoind signal handlers */ void noui_connect(); -/** Suppress all bitcoind signal handlers. Used to suppress output during test runs that produce expected errors */ -void noui_suppress(); +/** Redirect all bitcoind signal handlers to LogPrintf. Used to check or suppress output during test runs that produce expected errors */ +void noui_test_redirect(); -/** Reconnects the regular Non-GUI handlers after having used noui_suppress */ +/** Reconnects the regular Non-GUI handlers after having used noui_test_redirect */ void noui_reconnect(); #endif // BITCOIN_NOUI_H diff --git a/src/optional.h b/src/optional.h index 95a3b24d0a..a382cd7b77 100644 --- a/src/optional.h +++ b/src/optional.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 5cc43898a7..567eecb5c9 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +19,8 @@ static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; +const std::array<OutputType, 3> OUTPUT_TYPES = {OutputType::LEGACY, OutputType::P2SH_SEGWIT, OutputType::BECH32}; + bool ParseOutputType(const std::string& type, OutputType& output_type) { if (type == OUTPUT_TYPE_STRING_LEGACY) { @@ -80,22 +82,30 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, { // Add script to keystore keystore.AddCScript(script); + ScriptHash sh(script); // Note that scripts over 520 bytes are not yet supported. switch (type) { case OutputType::LEGACY: - return ScriptHash(script); + keystore.AddCScript(GetScriptForDestination(sh)); + return sh; case OutputType::P2SH_SEGWIT: case OutputType::BECH32: { CTxDestination witdest = WitnessV0ScriptHash(script); CScript witprog = GetScriptForDestination(witdest); // Check if the resulting program is solvable (i.e. doesn't use an uncompressed key) - if (!IsSolvable(keystore, witprog)) return ScriptHash(script); + if (!IsSolvable(keystore, witprog)) { + // Since the wsh is invalid, add and return the sh instead. + keystore.AddCScript(GetScriptForDestination(sh)); + return sh; + } // Add the redeemscript, so that P2WSH and P2SH-P2WSH outputs are recognized as ours. keystore.AddCScript(witprog); if (type == OutputType::BECH32) { return witdest; } else { - return ScriptHash(witprog); + ScriptHash sh_w = ScriptHash(witprog); + keystore.AddCScript(GetScriptForDestination(sh_w)); + return sh_w; } } default: assert(false); diff --git a/src/outputtype.h b/src/outputtype.h index 6acbaa2f3e..1438f65844 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +10,7 @@ #include <script/signingprovider.h> #include <script/standard.h> +#include <array> #include <string> #include <vector> @@ -27,6 +28,8 @@ enum class OutputType { CHANGE_AUTO, }; +extern const std::array<OutputType, 3> OUTPUT_TYPES; + NODISCARD bool ParseOutputType(const std::string& str, OutputType& output_type); const std::string& FormatOutputType(OutputType type); @@ -47,4 +50,3 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType); #endif // BITCOIN_OUTPUTTYPE_H - diff --git a/src/policy/feerate.h b/src/policy/feerate.h index d081f2ce8e..c040867965 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 8154bf105e..25458eead2 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -517,7 +517,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", - hash.ToString().c_str()); + hash.ToString()); return; } diff --git a/src/policy/fees.h b/src/policy/fees.h index 16683bf5ad..6ee6e0d547 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICY_FEES_H @@ -223,7 +223,7 @@ public: unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const; private: - mutable CCriticalSection m_cs_fee_estimator; + mutable RecursiveMutex m_cs_fee_estimator; unsigned int nBestSeenHeight GUARDED_BY(m_cs_fee_estimator); unsigned int firstRecordedHeight GUARDED_BY(m_cs_fee_estimator); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 51de5841ec..07d51c0088 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/policy.h b/src/policy/policy.h index ebe040f0ea..1561a41c5e 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index b4b8341d77..f8b17d18d5 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 0707b0044f..d335fbbb36 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/settings.cpp b/src/policy/settings.cpp index e8e1559407..eb2ec56850 100644 --- a/src/policy/settings.cpp +++ b/src/policy/settings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/settings.h b/src/policy/settings.h index 30a7189c93..0b4fc1e770 100644 --- a/src/policy/settings.h +++ b/src/policy/settings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/prevector.h b/src/prevector.h index d307495fbe..f4ece738a8 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,7 @@ #include <algorithm> #include <cstddef> #include <type_traits> +#include <utility> #pragma pack(push, 1) /** Implements a drop-in replacement for std::vector<T> which stores up to N diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 0c84ed6da2..50a30cb511 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index aad991e2f1..00ccbc32f9 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/protocol.h b/src/protocol.h index 3032310fa1..db07efb9f9 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -237,6 +237,7 @@ const std::vector<std::string> &getAllNetMessageTypes(); /** nServices flags */ enum ServiceFlags : uint64_t { + // NOTE: When adding here, be sure to update qt/guiutil.cpp's formatServicesStr too // Nothing NODE_NONE = 0, // NODE_NETWORK means that the node is capable of serving the complete block chain. It is currently diff --git a/src/psbt.cpp b/src/psbt.cpp index c306079b1e..e6b6285652 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -66,8 +66,11 @@ bool PartiallySignedTransaction::AddOutput(const CTxOut& txout, const PSBTOutput bool PartiallySignedTransaction::GetInputUTXO(CTxOut& utxo, int input_index) const { PSBTInput input = inputs[input_index]; - int prevout_index = tx->vin[input_index].prevout.n; + uint32_t prevout_index = tx->vin[input_index].prevout.n; if (input.non_witness_utxo) { + if (prevout_index >= input.non_witness_utxo->vout.size()) { + return false; + } utxo = input.non_witness_utxo->vout[prevout_index]; } else if (!input.witness_utxo.IsNull()) { utxo = input.witness_utxo; @@ -255,6 +258,9 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (input.non_witness_utxo) { // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. COutPoint prevout = tx.vin[index].prevout; + if (prevout.n >= input.non_witness_utxo->vout.size()) { + return false; + } if (input.non_witness_utxo->GetHash() != prevout.hash) { return false; } @@ -348,6 +354,7 @@ TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector std::string PSBTRoleName(PSBTRole role) { switch (role) { + case PSBTRole::CREATOR: return "creator"; case PSBTRole::UPDATER: return "updater"; case PSBTRole::SIGNER: return "signer"; case PSBTRole::FINALIZER: return "finalizer"; diff --git a/src/psbt.h b/src/psbt.h index 802a7c5ba7..dfba261961 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -171,7 +171,7 @@ struct PSBTInput case PSBT_IN_PARTIAL_SIG: { // Make sure that the key is the size of pubkey + 1 - if (key.size() != CPubKey::PUBLIC_KEY_SIZE + 1 && key.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1) { + if (key.size() != CPubKey::SIZE + 1 && key.size() != CPubKey::COMPRESSED_SIZE + 1) { throw std::ios_base::failure("Size of key was not the expected size for the type partial signature pubkey"); } // Read in the pubkey from key @@ -387,7 +387,7 @@ struct PSBTOutput /** A version of CTransaction with the PSBT format*/ struct PartiallySignedTransaction { - boost::optional<CMutableTransaction> tx; + Optional<CMutableTransaction> tx; std::vector<PSBTInput> inputs; std::vector<PSBTOutput> outputs; std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; @@ -401,7 +401,6 @@ struct PartiallySignedTransaction bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} - PartiallySignedTransaction(const PartiallySignedTransaction& psbt_in) : tx(psbt_in.tx), inputs(psbt_in.inputs), outputs(psbt_in.outputs), unknown(psbt_in.unknown) {} explicit PartiallySignedTransaction(const CMutableTransaction& tx); /** * Finds the UTXO for a given input index @@ -561,6 +560,7 @@ struct PartiallySignedTransaction }; enum class PSBTRole { + CREATOR, UPDATER, SIGNER, FINALIZER, @@ -584,7 +584,7 @@ void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransactio /** * Finalizes a PSBT if possible, combining partial signatures. * - * @param[in,out] &psbtx reference to PartiallySignedTransaction to finalize + * @param[in,out] psbtx PartiallySignedTransaction to finalize * return True if the PSBT is now complete, false otherwise */ bool FinalizePSBT(PartiallySignedTransaction& psbtx); @@ -592,7 +592,7 @@ bool FinalizePSBT(PartiallySignedTransaction& psbtx); /** * Finalizes a PSBT if possible, and extracts it to a CMutableTransaction if it could be finalized. * - * @param[in] &psbtx reference to PartiallySignedTransaction + * @param[in] psbtx PartiallySignedTransaction * @param[out] result CMutableTransaction representing the complete transaction, if successful * @return True if we successfully extracted the transaction, false otherwise */ @@ -601,7 +601,7 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti /** * Combines PSBTs with the same underlying transaction, resulting in a single PSBT with all partial signatures from each input. * - * @param[out] &out the combined PSBT, if successful + * @param[out] out the combined PSBT, if successful * @param[in] psbtxs the PSBTs to combine * @return error (OK if we successfully combined the transactions, other error if they were not compatible) */ diff --git a/src/pubkey.cpp b/src/pubkey.cpp index d38df716bd..ef42aa5bc7 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -171,6 +171,7 @@ bool CPubKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchS return false; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } @@ -190,14 +191,15 @@ bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned cha bool fComp = ((vchSig[0] - 27) & 4) != 0; secp256k1_pubkey pubkey; secp256k1_ecdsa_recoverable_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ecdsa_recoverable_signature_parse_compact(secp256k1_context_verify, &sig, &vchSig[1], recid)) { return false; } if (!secp256k1_ecdsa_recover(secp256k1_context_verify, &pubkey, &sig, hash.begin())) { return false; } - unsigned char pub[PUBLIC_KEY_SIZE]; - size_t publen = PUBLIC_KEY_SIZE; + unsigned char pub[SIZE]; + size_t publen = SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, fComp ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -207,6 +209,7 @@ bool CPubKey::IsFullyValid() const { if (!IsValid()) return false; secp256k1_pubkey pubkey; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); return secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size()); } @@ -214,11 +217,12 @@ bool CPubKey::Decompress() { if (!IsValid()) return false; secp256k1_pubkey pubkey; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } - unsigned char pub[PUBLIC_KEY_SIZE]; - size_t publen = PUBLIC_KEY_SIZE; + unsigned char pub[SIZE]; + size_t publen = SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -227,19 +231,20 @@ bool CPubKey::Decompress() { bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert((nChild >> 31) == 0); - assert(size() == COMPRESSED_PUBLIC_KEY_SIZE); + assert(size() == COMPRESSED_SIZE); unsigned char out[64]; BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); secp256k1_pubkey pubkey; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_verify, &pubkey, out)) { return false; } - unsigned char pub[COMPRESSED_PUBLIC_KEY_SIZE]; - size_t publen = COMPRESSED_PUBLIC_KEY_SIZE; + unsigned char pub[COMPRESSED_SIZE]; + size_t publen = COMPRESSED_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_COMPRESSED); pubkeyChild.Set(pub, pub + publen); return true; @@ -251,8 +256,8 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; memcpy(code+9, chaincode.begin(), 32); - assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); - memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); + assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); + memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_SIZE); } void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { @@ -273,6 +278,7 @@ bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { /* static */ bool CPubKey::CheckLowS(const std::vector<unsigned char>& vchSig) { secp256k1_ecdsa_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { return false; } diff --git a/src/pubkey.h b/src/pubkey.h index fd815a871b..2fc92c9bc6 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,17 +33,17 @@ public: /** * secp256k1: */ - static constexpr unsigned int PUBLIC_KEY_SIZE = 65; - static constexpr unsigned int COMPRESSED_PUBLIC_KEY_SIZE = 33; - static constexpr unsigned int SIGNATURE_SIZE = 72; - static constexpr unsigned int COMPACT_SIGNATURE_SIZE = 65; + static constexpr unsigned int SIZE = 65; + static constexpr unsigned int COMPRESSED_SIZE = 33; + static constexpr unsigned int SIGNATURE_SIZE = 72; + static constexpr unsigned int COMPACT_SIGNATURE_SIZE = 65; /** * see www.keylength.com * script supports up to 75 for single byte push */ static_assert( - PUBLIC_KEY_SIZE >= COMPRESSED_PUBLIC_KEY_SIZE, - "COMPRESSED_PUBLIC_KEY_SIZE is larger than PUBLIC_KEY_SIZE"); + SIZE >= COMPRESSED_SIZE, + "COMPRESSED_SIZE is larger than SIZE"); private: @@ -51,15 +51,15 @@ private: * Just store the serialized data. * Its length can very cheaply be computed from the first byte. */ - unsigned char vch[PUBLIC_KEY_SIZE]; + unsigned char vch[SIZE]; //! Compute the length of a pubkey with a given first byte. unsigned int static GetLen(unsigned char chHeader) { if (chHeader == 2 || chHeader == 3) - return COMPRESSED_PUBLIC_KEY_SIZE; + return COMPRESSED_SIZE; if (chHeader == 4 || chHeader == 6 || chHeader == 7) - return PUBLIC_KEY_SIZE; + return SIZE; return 0; } @@ -140,7 +140,7 @@ public: void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); - if (len <= PUBLIC_KEY_SIZE) { + if (len <= SIZE) { s.read((char*)vch, len); } else { // invalid pubkey, skip available data @@ -179,7 +179,7 @@ public: //! Check whether this is a compressed public key. bool IsCompressed() const { - return size() == COMPRESSED_PUBLIC_KEY_SIZE; + return size() == COMPRESSED_SIZE; } /** diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index d8c39e8862..1aaf33c6a4 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 131cceccbe..3ac98a5970 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index 2ababb5e1e..67e7704551 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index bdfd3fb9a0..20fc5045ae 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index ae11b80347..72f16bb09f 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -1,17 +1,18 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/bantablemodel.h> -#include <qt/clientmodel.h> - #include <interfaces/node.h> +#include <net_types.h> // For banmap_t -#include <algorithm> +#include <utility> -#include <QDebug> +#include <QDateTime> #include <QList> +#include <QModelIndex> +#include <QVariant> bool BannedNodeLessThan::operator()(const CCombinedBan& left, const CCombinedBan& right) const { @@ -78,10 +79,9 @@ public: } }; -BanTableModel::BanTableModel(interfaces::Node& node, ClientModel *parent) : +BanTableModel::BanTableModel(interfaces::Node& node, QObject* parent) : QAbstractTableModel(parent), - m_node(node), - clientModel(parent) + m_node(node) { columns << tr("IP/Netmask") << tr("Banned Until"); priv.reset(new BanTablePriv()); diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 9dec5fa6a9..f01c506a1e 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -12,7 +12,6 @@ #include <QAbstractTableModel> #include <QStringList> -class ClientModel; class BanTablePriv; namespace interfaces { @@ -45,7 +44,7 @@ class BanTableModel : public QAbstractTableModel Q_OBJECT public: - explicit BanTableModel(interfaces::Node& node, ClientModel *parent = nullptr); + explicit BanTableModel(interfaces::Node& node, QObject* parent); ~BanTableModel(); void startAutoRefresh(); void stopAutoRefresh(); @@ -72,7 +71,6 @@ public Q_SLOTS: private: interfaces::Node& m_node; - ClientModel *clientModel; QStringList columns; std::unique_ptr<BanTablePriv> priv; }; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 02a2a01bdd..4313d6ee7f 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -135,7 +135,7 @@ BitcoinCore::BitcoinCore(interfaces::Node& node) : void BitcoinCore::handleRunawayException(const std::exception *e) { PrintExceptionContinue(e, "Runaway exception"); - Q_EMIT runawayException(QString::fromStdString(m_node.getWarnings("gui"))); + Q_EMIT runawayException(QString::fromStdString(m_node.getWarnings())); } void BitcoinCore::initialize() @@ -281,8 +281,11 @@ void BitcoinApplication::parameterSetup() m_node.initParameterInteraction(); } -void BitcoinApplication::SetPrune(bool prune, bool force) { - optionsModel->SetPrune(prune, force); +void BitcoinApplication::InitializePruneSetting(bool prune) +{ + // If prune is set, intentionally override existing prune size with + // the default size since this is called when choosing a new datadir. + optionsModel->SetPruneTargetGB(prune ? DEFAULT_PRUNE_TARGET_GB : 0, true); } void BitcoinApplication::requestInitialize() @@ -309,7 +312,7 @@ void BitcoinApplication::requestShutdown() // rescanning a wallet. m_node.startShutdown(); // Unsetting the client model can cause the current thread to wait for node - // to complete an operation, like wait for a RPC execution to complate. + // to complete an operation, like wait for a RPC execution to complete. window->setClientModel(nullptr); pollShutdownTimer->stop(); @@ -430,16 +433,19 @@ int GuiMain(int argc, char* argv[]) BitcoinApplication app(*node); - // Register meta types used for QMetaObject::invokeMethod - qRegisterMetaType< bool* >(); + // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection + qRegisterMetaType<bool*>(); #ifdef ENABLE_WALLET qRegisterMetaType<WalletModel*>(); #endif - // Need to pass name here as CAmount is a typedef (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) - // IMPORTANT if it is no longer a typedef use the normal variant above - qRegisterMetaType< CAmount >("CAmount"); - qRegisterMetaType< std::function<void()> >("std::function<void()>"); + // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) + // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) + qRegisterMetaType<CAmount>("CAmount"); + qRegisterMetaType<size_t>("size_t"); + + qRegisterMetaType<std::function<void()>>("std::function<void()>"); qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); + /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: node->setupServerArgs(); @@ -553,12 +559,13 @@ int GuiMain(int argc, char* argv[]) qInstallMessageHandler(DebugMessageHandler); // Allow parameter interaction before we create the options model app.parameterSetup(); + GUIUtil::LogQtInfo(); // Load GUI settings from QSettings app.createOptionsModel(gArgs.GetBoolArg("-resetguisettings", false)); if (did_show_intro) { // Store intro dialog settings other than datadir (network specific) - app.SetPrune(prune, true); + app.InitializePruneSetting(prune); } if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) @@ -586,10 +593,10 @@ int GuiMain(int argc, char* argv[]) } } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(node->getWarnings("gui"))); + app.handleRunawayException(QString::fromStdString(node->getWarnings())); } catch (...) { PrintExceptionContinue(nullptr, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(node->getWarnings("gui"))); + app.handleRunawayException(QString::fromStdString(node->getWarnings())); } return rv; } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 8c77fd8a7d..077a37fde5 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -67,8 +67,8 @@ public: void parameterSetup(); /// Create options model void createOptionsModel(bool resetSettings); - /// Update prune value - void SetPrune(bool prune, bool force = false); + /// Initialize prune setting + void InitializePruneSetting(bool prune); /// Create main window void createWindow(const NetworkStyle *networkStyle); /// Create splash screen diff --git a/src/qt/bitcoin_locale.qrc b/src/qt/bitcoin_locale.qrc index dec3670536..c781072e9b 100644 --- a/src/qt/bitcoin_locale.qrc +++ b/src/qt/bitcoin_locale.qrc @@ -11,6 +11,7 @@ <file alias="de_DE">locale/bitcoin_de_DE.qm</file> <file alias="el">locale/bitcoin_el.qm</file> <file alias="el_GR">locale/bitcoin_el_GR.qm</file> + <file alias="en">locale/bitcoin_en.qm</file> <file alias="en_AU">locale/bitcoin_en_AU.qm</file> <file alias="en_GB">locale/bitcoin_en_GB.qm</file> <file alias="eo">locale/bitcoin_eo.qm</file> diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 23df1c929a..7acc82370f 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 2aeba6d82c..5fab267610 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -318,8 +318,8 @@ void BitcoinGUI::createActions() verifyMessageAction = new QAction(tr("&Verify message..."), this); verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); - openRPCConsoleAction = new QAction(tr("&Debug window"), this); - openRPCConsoleAction->setStatusTip(tr("Open debugging and diagnostic console")); + openRPCConsoleAction = new QAction(tr("Node window"), this); + openRPCConsoleAction->setStatusTip(tr("Open node debugging and diagnostic console")); // initially disable the debug window menu item openRPCConsoleAction->setEnabled(false); openRPCConsoleAction->setObjectName("openRPCConsoleAction"); @@ -341,6 +341,7 @@ void BitcoinGUI::createActions() m_close_wallet_action->setStatusTip(tr("Close wallet")); m_create_wallet_action = new QAction(tr("Create Wallet..."), this); + m_create_wallet_action->setEnabled(false); m_create_wallet_action->setStatusTip(tr("Create a new wallet")); showHelpMessageAction = new QAction(tr("&Command-line options"), this); @@ -618,6 +619,7 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) m_wallet_controller = wallet_controller; + m_create_wallet_action->setEnabled(true); m_open_wallet_action->setEnabled(true); m_open_wallet_action->setMenu(m_open_wallet_menu); @@ -632,10 +634,10 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) void BitcoinGUI::addWallet(WalletModel* walletModel) { if (!walletFrame) return; + if (!walletFrame->addWallet(walletModel)) return; const QString display_name = walletModel->getDisplayName(); setWalletActionsEnabled(true); rpcConsole->addWallet(walletModel); - walletFrame->addWallet(walletModel); m_wallet_selector->addItem(display_name, QVariant::fromValue(walletModel)); if (m_wallet_selector->count() == 2) { m_wallet_selector_label_action->setVisible(true); @@ -646,6 +648,10 @@ void BitcoinGUI::addWallet(WalletModel* walletModel) void BitcoinGUI::removeWallet(WalletModel* walletModel) { if (!walletFrame) return; + + labelWalletHDStatusIcon->hide(); + labelWalletEncryptionIcon->hide(); + int index = m_wallet_selector->findData(QVariant::fromValue(walletModel)); m_wallet_selector->removeItem(index); if (m_wallet_selector->count() == 0) { @@ -1207,7 +1213,7 @@ void BitcoinGUI::setHDStatus(bool privkeyDisabled, int hdEnabled) { labelWalletHDStatusIcon->setPixmap(platformStyle->SingleColorIcon(privkeyDisabled ? ":/icons/eye" : hdEnabled ? ":/icons/hd_enabled" : ":/icons/hd_disabled").pixmap(STATUSBAR_ICONSIZE,STATUSBAR_ICONSIZE)); labelWalletHDStatusIcon->setToolTip(privkeyDisabled ? tr("Private key <b>disabled</b>") : hdEnabled ? tr("HD key generation is <b>enabled</b>") : tr("HD key generation is <b>disabled</b>")); - + labelWalletHDStatusIcon->show(); // eventually disable the QLabel to set its opacity to 50% labelWalletHDStatusIcon->setEnabled(hdEnabled); } diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index b27f8a744f..d9711af123 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 06a1544fa2..4c8a889965 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,22 +13,6 @@ // U+2009 THIN SPACE = UTF-8 E2 80 89 #define REAL_THIN_SP_CP 0x2009 #define REAL_THIN_SP_UTF8 "\xE2\x80\x89" -#define REAL_THIN_SP_HTML " " - -// U+200A HAIR SPACE = UTF-8 E2 80 8A -#define HAIR_SP_CP 0x200A -#define HAIR_SP_UTF8 "\xE2\x80\x8A" -#define HAIR_SP_HTML " " - -// U+2006 SIX-PER-EM SPACE = UTF-8 E2 80 86 -#define SIXPEREM_SP_CP 0x2006 -#define SIXPEREM_SP_UTF8 "\xE2\x80\x86" -#define SIXPEREM_SP_HTML " " - -// U+2007 FIGURE SPACE = UTF-8 E2 80 87 -#define FIGURE_SP_CP 0x2007 -#define FIGURE_SP_UTF8 "\xE2\x80\x87" -#define FIGURE_SP_HTML " " // QMessageBox seems to have a bug whereby it doesn't display thin/hair spaces // correctly. Workaround is to display a space in a small font. If you @@ -114,9 +98,6 @@ public: { text.remove(' '); text.remove(QChar(THIN_SP_CP)); -#if (THIN_SP_CP != REAL_THIN_SP_CP) - text.remove(QChar(REAL_THIN_SP_CP)); -#endif return text; } diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 5b216b2705..e8146982f9 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -134,7 +134,7 @@ enum BlockSource ClientModel::getBlockSource() const QString ClientModel::getStatusBarWarnings() const { - return QString::fromStdString(m_node.getWarnings("gui")); + return QString::fromStdString(m_node.getWarnings()); } OptionsModel *ClientModel::getOptionsModel() diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index d3a95d531e..79175e0af4 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index f928f1ca2a..9495ba389a 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -468,7 +468,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) { CPubKey pubkey; PKHash *pkhash = boost::get<PKHash>(&address); - if (pkhash && model->wallet().getPubKey(CKeyID(*pkhash), pubkey)) + if (pkhash && model->wallet().getPubKey(out.txout.scriptPubKey, CKeyID(*pkhash), pubkey)) { nBytesInputs += (pubkey.IsCompressed() ? 148 : 180); } diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 88fc8b704f..39dc9a5e9e 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/csvmodelwriter.h b/src/qt/csvmodelwriter.h index e8611bea35..e443529335 100644 --- a/src/qt/csvmodelwriter.h +++ b/src/qt/csvmodelwriter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index be807b20c0..ebb6bbd4f5 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -11,7 +11,7 @@ </rect> </property> <property name="windowTitle"> - <string>Debug window</string> + <string>Node window</string> </property> <layout class="QVBoxLayout" name="verticalLayout_2"> <item> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index b5a69c578d..d2e7ca8f06 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <ui version="4.0"> <class>ModalOverlay</class> - <widget class="ModalOverlay" name="ModalOverlay"> + <widget class="QWidget" name="ModalOverlay"> <property name="geometry"> <rect> <x>0</x> @@ -351,6 +351,9 @@ QLabel { color: rgb(40,40,40); }</string> <property name="text"> <string>Hide</string> </property> + <property name="shortcut"> + <string>Esc</string> + </property> <property name="focusPolicy"> <enum>Qt::StrongFocus</enum> </property> @@ -369,14 +372,6 @@ QLabel { color: rgb(40,40,40); }</string> </item> </layout> </widget> - <customwidgets> - <customwidget> - <class>ModalOverlay</class> - <extends>QWidget</extends> - <header>qt/modaloverlay.h</header> - <container>1</container> - </customwidget> - </customwidgets> <resources/> <connections/> </ui> diff --git a/src/qt/forms/openuridialog.ui b/src/qt/forms/openuridialog.ui index 2acec314fd..1b7291ab9d 100644 --- a/src/qt/forms/openuridialog.ui +++ b/src/qt/forms/openuridialog.ui @@ -24,7 +24,11 @@ </widget> </item> <item> - <widget class="QValidatedLineEdit" name="uriEdit"/> + <widget class="QValidatedLineEdit" name="uriEdit"> + <property name="placeholderText"> + <string notr="true">bitcoin:</string> + </property> + </widget> </item> </layout> </item> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 240a7a7e92..fea759dee0 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -685,6 +685,9 @@ <property name="toolTip"> <string>Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> </property> + <property name="placeholderText"> + <string notr="true">https://example.com/tx/%s</string> + </property> </widget> </item> </layout> diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 0214356eaa..7dbee6d689 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <ui version="4.0"> <class>ReceiveCoinsDialog</class> - <widget class="QWidget" name="ReceiveCoinsDialog"> + <widget class="QDialog" name="ReceiveCoinsDialog"> <property name="geometry"> <rect> <x>0</x> @@ -63,7 +63,7 @@ <item row="4" column="2"> <widget class="QLineEdit" name="reqLabel"> <property name="toolTip"> - <string>An optional label to associate with the new receiving address.</string> + <string>An optional label to associate with the new receiving address (used by you to identify an invoice). It is also attached to the payment request.</string> </property> </widget> </item> @@ -93,7 +93,7 @@ <item row="6" column="2"> <widget class="QLineEdit" name="reqMessage"> <property name="toolTip"> - <string>An optional message to attach to the payment request, which will be displayed when the request is opened. Note: The message will not be sent with the payment over the Bitcoin network.</string> + <string>An optional message that is attached to the payment request and may be displayed to the sender.</string> </property> </widget> </item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 386d559281..cfd4bf33d4 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -797,7 +797,7 @@ <item> <widget class="QPushButton" name="buttonMinimizeFee"> <property name="toolTip"> - <string>collapse fee-settings</string> + <string>Hide transaction fee settings</string> </property> <property name="text"> <string>Hide</string> @@ -1190,7 +1190,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee of "100 satoshis p <number>3</number> </property> <item> - <widget class="QLabel" name="label"> + <widget class="QLabel" name="labelBalanceName"> <property name="sizePolicy"> <sizepolicy hsizetype="Preferred" vsizetype="Fixed"> <horstretch>0</horstretch> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index 843d909f68..934363af1f 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -144,6 +144,9 @@ <property name="toolTip"> <string>Enter a label for this address to add it to the list of used addresses</string> </property> + <property name="placeholderText"> + <string>Enter a label for this address to add it to the list of used addresses</string> + </property> </widget> </item> <item row="2" column="0"> diff --git a/src/qt/forms/signverifymessagedialog.ui b/src/qt/forms/signverifymessagedialog.ui index 202edf27d4..f42d19093b 100644 --- a/src/qt/forms/signverifymessagedialog.ui +++ b/src/qt/forms/signverifymessagedialog.ui @@ -121,6 +121,9 @@ </property> <item> <widget class="QLineEdit" name="signatureOut_SM"> + <property name="placeholderText"> + <string>Click "Sign Message" to generate signature</string> + </property> <property name="font"> <font> <italic>true</italic> diff --git a/src/qt/guiconstants.h b/src/qt/guiconstants.h index dcdb247977..9457ea37d6 100644 --- a/src/qt/guiconstants.h +++ b/src/qt/guiconstants.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -51,4 +51,7 @@ static const int TOOLTIP_WRAP_THRESHOLD = 80; /* One gigabyte (GB) in bytes */ static constexpr uint64_t GB_BYTES{1000000000}; +// Default prune target displayed in GUI. +static constexpr int DEFAULT_PRUNE_TARGET_GB{2}; + #endif // BITCOIN_QT_GUICONSTANTS_H diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index c4e0321f28..911322092c 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -7,7 +7,7 @@ #include <qt/bitcoinaddressvalidator.h> #include <qt/bitcoinunits.h> #include <qt/qvalidatedlineedit.h> -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <base58.h> #include <chainparams.h> @@ -44,20 +44,23 @@ #include <QFont> #include <QFontDatabase> #include <QFontMetrics> +#include <QGuiApplication> #include <QKeyEvent> #include <QLineEdit> +#include <QList> #include <QMouseEvent> #include <QProgressDialog> +#include <QScreen> #include <QSettings> +#include <QSize> +#include <QString> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> #include <QUrlQuery> +#include <QtGlobal> #if defined(Q_OS_MAC) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include <CoreServices/CoreServices.h> #include <QProcess> void ForceActivation(); @@ -691,87 +694,6 @@ bool SetStartOnSystemStartup(bool fAutoStart) return true; } - -#elif defined(Q_OS_MAC) && defined(MAC_OS_X_VERSION_MIN_REQUIRED) && MAC_OS_X_VERSION_MIN_REQUIRED <= 101100 -// based on: https://github.com/Mozketo/LaunchAtLoginController/blob/master/LaunchAtLoginController.m - -LSSharedFileListItemRef findStartupItemInList(CFArrayRef listSnapshot, LSSharedFileListRef list, CFURLRef findUrl) -{ - if (listSnapshot == nullptr) { - return nullptr; - } - - // loop through the list of startup items and try to find the bitcoin app - for(int i = 0; i < CFArrayGetCount(listSnapshot); i++) { - LSSharedFileListItemRef item = (LSSharedFileListItemRef)CFArrayGetValueAtIndex(listSnapshot, i); - UInt32 resolutionFlags = kLSSharedFileListNoUserInteraction | kLSSharedFileListDoNotMountVolumes; - CFURLRef currentItemURL = nullptr; - -#if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED >= 10100 - if(&LSSharedFileListItemCopyResolvedURL) - currentItemURL = LSSharedFileListItemCopyResolvedURL(item, resolutionFlags, nullptr); -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && MAC_OS_X_VERSION_MIN_REQUIRED < 10100 - else - LSSharedFileListItemResolve(item, resolutionFlags, ¤tItemURL, nullptr); -#endif -#else - LSSharedFileListItemResolve(item, resolutionFlags, ¤tItemURL, nullptr); -#endif - - if(currentItemURL) { - if (CFEqual(currentItemURL, findUrl)) { - // found - CFRelease(currentItemURL); - return item; - } - CFRelease(currentItemURL); - } - } - return nullptr; -} - -bool GetStartOnSystemStartup() -{ - CFURLRef bitcoinAppUrl = CFBundleCopyBundleURL(CFBundleGetMainBundle()); - if (bitcoinAppUrl == nullptr) { - return false; - } - - LSSharedFileListRef loginItems = LSSharedFileListCreate(nullptr, kLSSharedFileListSessionLoginItems, nullptr); - CFArrayRef listSnapshot = LSSharedFileListCopySnapshot(loginItems, nullptr); - bool res = (findStartupItemInList(listSnapshot, loginItems, bitcoinAppUrl) != nullptr); - CFRelease(bitcoinAppUrl); - CFRelease(loginItems); - CFRelease(listSnapshot); - return res; -} - -bool SetStartOnSystemStartup(bool fAutoStart) -{ - CFURLRef bitcoinAppUrl = CFBundleCopyBundleURL(CFBundleGetMainBundle()); - if (bitcoinAppUrl == nullptr) { - return false; - } - - LSSharedFileListRef loginItems = LSSharedFileListCreate(nullptr, kLSSharedFileListSessionLoginItems, nullptr); - CFArrayRef listSnapshot = LSSharedFileListCopySnapshot(loginItems, nullptr); - LSSharedFileListItemRef foundItem = findStartupItemInList(listSnapshot, loginItems, bitcoinAppUrl); - - if(fAutoStart && !foundItem) { - // add bitcoin app to startup item list - LSSharedFileListInsertItemURL(loginItems, kLSSharedFileListItemBeforeFirst, nullptr, nullptr, bitcoinAppUrl, nullptr, nullptr); - } - else if(!fAutoStart && foundItem) { - // remove item - LSSharedFileListItemRemove(loginItems, foundItem); - } - - CFRelease(bitcoinAppUrl); - CFRelease(loginItems); - CFRelease(listSnapshot); - return true; -} -#pragma GCC diagnostic pop #else bool GetStartOnSystemStartup() { return false; } @@ -815,32 +737,33 @@ QString formatDurationStr(int secs) return strList.join(" "); } +QString serviceFlagToStr(const quint64 mask, const int bit) +{ + switch (ServiceFlags(mask)) { + case NODE_NONE: abort(); // impossible + case NODE_NETWORK: return "NETWORK"; + case NODE_GETUTXO: return "GETUTXO"; + case NODE_BLOOM: return "BLOOM"; + case NODE_WITNESS: return "WITNESS"; + case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED"; + // Not using default, so we get warned when a case is missing + } + if (bit < 8) { + return QString("%1[%2]").arg("UNKNOWN").arg(mask); + } else { + return QString("%1[2^%2]").arg("UNKNOWN").arg(bit); + } +} + QString formatServicesStr(quint64 mask) { QStringList strList; - // Just scan the last 8 bits for now. - for (int i = 0; i < 8; i++) { - uint64_t check = 1 << i; + for (int i = 0; i < 64; i++) { + uint64_t check = 1LL << i; if (mask & check) { - switch (check) - { - case NODE_NETWORK: - strList.append("NETWORK"); - break; - case NODE_GETUTXO: - strList.append("GETUTXO"); - break; - case NODE_BLOOM: - strList.append("BLOOM"); - break; - case NODE_WITNESS: - strList.append("WITNESS"); - break; - default: - strList.append(QString("%1[%2]").arg("UNKNOWN").arg(check)); - } + strList.append(serviceFlagToStr(check, i)); } } @@ -962,4 +885,23 @@ int TextWidth(const QFontMetrics& fm, const QString& text) #endif } +void LogQtInfo() +{ +#ifdef QT_STATIC + const std::string qt_link{"static"}; +#else + const std::string qt_link{"dynamic"}; +#endif +#ifdef QT_STATICPLUGIN + const std::string plugin_link{"static"}; +#else + const std::string plugin_link{"dynamic"}; +#endif + LogPrintf("Qt %s (%s), plugin=%s (%s)\n", qVersion(), qt_link, QGuiApplication::platformName().toStdString(), plugin_link); + LogPrintf("System: %s, %s\n", QSysInfo::prettyProductName().toStdString(), QSysInfo::buildAbi().toStdString()); + for (const QScreen* s : QGuiApplication::screens()) { + LogPrintf("Screen: %s %dx%d, pixel ratio=%.1f\n", s->name().toStdString(), s->size().width(), s->size().height(), s->devicePixelRatio()); + } +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 9db92f94d7..05e73cc5f0 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -265,6 +265,11 @@ namespace GUIUtil * In Qt 5.11 the QFontMetrics::horizontalAdvance() was introduced. */ int TextWidth(const QFontMetrics& fm, const QString& text); + + /** + * Writes to debug.log short info about the used Qt and the host system. + */ + void LogQtInfo(); } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 53c80639b9..ad21dfc3ef 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,7 @@ #include <qt/guiconstants.h> #include <qt/guiutil.h> +#include <qt/optionsmodel.h> #include <interfaces/node.h> #include <util/system.h> @@ -22,9 +23,6 @@ #include <cmath> -/* Total required space (in GB) depending on user choice (prune, not prune) */ -static uint64_t requiredSpace; - /* Check free space asynchronously to prevent hanging the UI thread. Up to one request to check a path is in flight to this thread; when the check() @@ -109,14 +107,24 @@ void FreespaceChecker::check() Q_EMIT reply(replyStatus, replyMessage, freeBytesAvailable); } +namespace { +//! Return pruning size that will be used if automatic pruning is enabled. +int GetPruneTargetGB() +{ + int64_t prune_target_mib = gArgs.GetArg("-prune", 0); + // >1 means automatic pruning is enabled by config, 1 means manual pruning, 0 means no pruning. + return prune_target_mib > 1 ? PruneMiBtoGB(prune_target_mib) : DEFAULT_PRUNE_TARGET_GB; +} +} // namespace -Intro::Intro(QWidget *parent, uint64_t blockchain_size, uint64_t chain_state_size) : +Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : QDialog(parent), ui(new Ui::Intro), thread(nullptr), signalled(false), - m_blockchain_size(blockchain_size), - m_chain_state_size(chain_state_size) + m_blockchain_size_gb(blockchain_size_gb), + m_chain_state_size_gb(chain_state_size_gb), + m_prune_target_gb{GetPruneTargetGB()} { ui->setupUi(this); ui->welcomeLabel->setText(ui->welcomeLabel->text().arg(PACKAGE_NAME)); @@ -124,37 +132,24 @@ Intro::Intro(QWidget *parent, uint64_t blockchain_size, uint64_t chain_state_siz ui->lblExplanation1->setText(ui->lblExplanation1->text() .arg(PACKAGE_NAME) - .arg(m_blockchain_size) + .arg(m_blockchain_size_gb) .arg(2009) .arg(tr("Bitcoin")) ); ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(PACKAGE_NAME)); - uint64_t pruneTarget = std::max<int64_t>(0, gArgs.GetArg("-prune", 0)); - if (pruneTarget > 1) { // -prune=1 means enabled, above that it's a size in MB + if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } - ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(pruneTarget ? pruneTarget / 1000 : 2)); - requiredSpace = m_blockchain_size; - QString storageRequiresMsg = tr("At least %1 GB of data will be stored in this directory, and it will grow over time."); - if (pruneTarget) { - uint64_t prunedGBs = std::ceil(pruneTarget * 1024 * 1024.0 / GB_BYTES); - if (prunedGBs <= requiredSpace) { - requiredSpace = prunedGBs; - storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); - } - ui->lblExplanation3->setVisible(true); - } else { - ui->lblExplanation3->setVisible(false); - } - requiredSpace += m_chain_state_size; - ui->sizeWarningLabel->setText( - tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + - storageRequiresMsg.arg(requiredSpace) + " " + - tr("The wallet will also be stored in this directory.") - ); - this->adjustSize(); + ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(m_prune_target_gb)); + UpdatePruneLabels(ui->prune->isChecked()); + + connect(ui->prune, &QCheckBox::toggled, [this](bool prune_checked) { + UpdatePruneLabels(prune_checked); + UpdateFreeSpaceLabel(); + }); + startThread(); } @@ -270,25 +265,31 @@ void Intro::setStatus(int status, const QString &message, quint64 bytesAvailable { ui->freeSpace->setText(""); } else { - QString freeString = tr("%n GB of free space available", "", bytesAvailable/GB_BYTES); - if(bytesAvailable < requiredSpace * GB_BYTES) - { - freeString += " " + tr("(of %n GB needed)", "", requiredSpace); - ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); - ui->prune->setChecked(true); - } else if (bytesAvailable / GB_BYTES - requiredSpace < 10) { - freeString += " " + tr("(%n GB needed for full chain)", "", requiredSpace); - ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); - ui->prune->setChecked(true); - } else { - ui->freeSpace->setStyleSheet(""); + m_bytes_available = bytesAvailable; + if (ui->prune->isEnabled()) { + ui->prune->setChecked(m_bytes_available < (m_blockchain_size_gb + m_chain_state_size_gb + 10) * GB_BYTES); } - ui->freeSpace->setText(freeString + "."); + UpdateFreeSpaceLabel(); } /* Don't allow confirm in ERROR state */ ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(status != FreespaceChecker::ST_ERROR); } +void Intro::UpdateFreeSpaceLabel() +{ + QString freeString = tr("%n GB of free space available", "", m_bytes_available / GB_BYTES); + if (m_bytes_available < m_required_space_gb * GB_BYTES) { + freeString += " " + tr("(of %n GB needed)", "", m_required_space_gb); + ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); + } else if (m_bytes_available / GB_BYTES - m_required_space_gb < 10) { + freeString += " " + tr("(%n GB needed for full chain)", "", m_required_space_gb); + ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); + } else { + ui->freeSpace->setStyleSheet(""); + } + ui->freeSpace->setText(freeString + "."); +} + void Intro::on_dataDirectory_textChanged(const QString &dataDirStr) { /* Disable OK button until check result comes in */ @@ -349,3 +350,20 @@ QString Intro::getPathToCheck() mutex.unlock(); return retval; } + +void Intro::UpdatePruneLabels(bool prune_checked) +{ + m_required_space_gb = m_blockchain_size_gb + m_chain_state_size_gb; + QString storageRequiresMsg = tr("At least %1 GB of data will be stored in this directory, and it will grow over time."); + if (prune_checked && m_prune_target_gb <= m_blockchain_size_gb) { + m_required_space_gb = m_prune_target_gb + m_chain_state_size_gb; + storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); + } + ui->lblExplanation3->setVisible(prune_checked); + ui->sizeWarningLabel->setText( + tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + + storageRequiresMsg.arg(m_required_space_gb) + " " + + tr("The wallet will also be stored in this directory.") + ); + this->adjustSize(); +} diff --git a/src/qt/intro.h b/src/qt/intro.h index aca7e71642..732393246e 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,7 @@ class Intro : public QDialog public: explicit Intro(QWidget *parent = nullptr, - uint64_t blockchain_size = 0, uint64_t chain_state_size = 0); + int64_t blockchain_size_gb = 0, int64_t chain_state_size_gb = 0); ~Intro(); QString getDataDirectory(); @@ -67,12 +67,18 @@ private: QMutex mutex; bool signalled; QString pathToCheck; - uint64_t m_blockchain_size; - uint64_t m_chain_state_size; + const int64_t m_blockchain_size_gb; + const int64_t m_chain_state_size_gb; + //! Total required space (in GB) depending on user choice (prune or not prune). + int64_t m_required_space_gb{0}; + uint64_t m_bytes_available{0}; + const int64_t m_prune_target_gb; void startThread(); void checkPath(const QString &dataDir); QString getPathToCheck(); + void UpdatePruneLabels(bool prune_checked); + void UpdateFreeSpaceLabel(); friend class FreespaceChecker; }; diff --git a/src/qt/macnotificationhandler.mm b/src/qt/macnotificationhandler.mm index a07079eece..b16042e946 100644 --- a/src/qt/macnotificationhandler.mm +++ b/src/qt/macnotificationhandler.mm @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2013 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 999c434d23..3dfd9e850e 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index efdd494d9f..6243a71c7d 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,7 @@ userClosed(false) setVisible(false); if (!enable_wallet) { ui->infoText->setVisible(false); - ui->infoTextStrong->setText(tr("Bitcoin Core is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.")); + ui->infoTextStrong->setText(tr("%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.").arg(PACKAGE_NAME)); } } diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index c075a89f94..076ec30b58 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index 5c039a939e..3a251e0573 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/networkstyle.h b/src/qt/networkstyle.h index bb12dd1b6e..a73e3e2625 100644 --- a/src/qt/networkstyle.h +++ b/src/qt/networkstyle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,7 @@ class NetworkStyle { public: - /** Get style associated with provided BIP70 network id, or 0 if not known */ + /** Get style associated with provided network id, or 0 if not known */ static const NetworkStyle* instantiate(const std::string& networkId); const QString &getAppName() const { return appName; } diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 199804f84d..b9dea2f8bf 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include <qt/forms/ui_openuridialog.h> #include <qt/guiutil.h> -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <QUrl> @@ -15,7 +15,6 @@ OpenURIDialog::OpenURIDialog(QWidget *parent) : ui(new Ui::OpenURIDialog) { ui->setupUi(this); - ui->uriEdit->setPlaceholderText("bitcoin:"); } OpenURIDialog::~OpenURIDialog() diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index 8438f22bd7..4b610f74d7 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 57cafaaac0..8ee6c947e6 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -71,17 +71,17 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : #ifdef Q_OS_MAC /* remove Window tab on Mac */ ui->tabWidget->removeTab(ui->tabWidget->indexOf(ui->tabWindow)); -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && MAC_OS_X_VERSION_MIN_REQUIRED > 101100 - /* hide launch at startup option if compiled against macOS > 10.11 (removed API) */ + /* hide launch at startup option on macOS */ ui->bitcoinAtStartup->setVisible(false); ui->verticalLayout_Main->removeWidget(ui->bitcoinAtStartup); ui->verticalLayout_Main->removeItem(ui->horizontalSpacer_0_Main); #endif -#endif - /* remove Wallet tab in case of -disablewallet */ + /* remove Wallet tab and 3rd party-URL textbox in case of -disablewallet */ if (!enableWallet) { ui->tabWidget->removeTab(ui->tabWidget->indexOf(ui->tabWallet)); + ui->thirdPartyTxUrlsLabel->setVisible(false); + ui->thirdPartyTxUrls->setVisible(false); } /* Display elements init */ @@ -110,8 +110,6 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->lang->addItem(locale.nativeLanguageName() + QString(" (") + langStr + QString(")"), QVariant(langStr)); } } - ui->thirdPartyTxUrls->setPlaceholderText("https://example.com/tx/%s"); - ui->unit->setModel(new BitcoinUnits(this)); /* Widget-to-option mapper */ @@ -377,7 +375,7 @@ QValidator::State ProxyAddressValidator::validate(QString &input, int &pos) cons { Q_UNUSED(pos); // Validate the proxy - CService serv(LookupNumeric(input.toStdString().c_str(), DEFAULT_GUI_PROXY_PORT)); + CService serv(LookupNumeric(input.toStdString(), DEFAULT_GUI_PROXY_PORT)); proxyType addrProxy = proxyType(serv, true); if (addrProxy.IsValid()) return QValidator::Acceptable; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d74d0dbfeb..977076c4c2 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -91,8 +91,8 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("bPrune")) settings.setValue("bPrune", false); if (!settings.contains("nPruneSize")) - settings.setValue("nPruneSize", 2); - SetPrune(settings.value("bPrune").toBool()); + settings.setValue("nPruneSize", DEFAULT_PRUNE_TARGET_GB); + SetPruneEnabled(settings.value("bPrune").toBool()); if (!settings.contains("nDatabaseCache")) settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache); @@ -236,13 +236,12 @@ static const QString GetDefaultProxyAddress() return QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST).arg(DEFAULT_GUI_PROXY_PORT); } -void OptionsModel::SetPrune(bool prune, bool force) +void OptionsModel::SetPruneEnabled(bool prune, bool force) { QSettings settings; settings.setValue("bPrune", prune); - // Convert prune size from GB to MiB: - const uint64_t nPruneSizeMiB = (settings.value("nPruneSize").toInt() * GB_BYTES) >> 20; - std::string prune_val = prune ? std::to_string(nPruneSizeMiB) : "0"; + const int64_t prune_target_mib = PruneGBtoMiB(settings.value("nPruneSize").toInt()); + std::string prune_val = prune ? std::to_string(prune_target_mib) : "0"; if (force) { m_node.forceSetArg("-prune", prune_val); return; @@ -252,6 +251,16 @@ void OptionsModel::SetPrune(bool prune, bool force) } } +void OptionsModel::SetPruneTargetGB(int prune_target_gb, bool force) +{ + const bool prune = prune_target_gb > 0; + if (prune) { + QSettings settings; + settings.setValue("nPruneSize", prune_target_gb); + } + SetPruneEnabled(prune, force); +} + // read QSettings values and return them QVariant OptionsModel::data(const QModelIndex & index, int role) const { diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 5791b47f28..b3260349e7 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include <amount.h> +#include <qt/guiconstants.h> #include <QAbstractListModel> @@ -16,6 +17,16 @@ class Node; extern const char *DEFAULT_GUI_PROXY_HOST; static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; +/** + * Convert configured prune target MiB to displayed GB. Round up to avoid underestimating max disk usage. + */ +static inline int PruneMiBtoGB(int64_t mib) { return (mib * 1024 * 1024 + GB_BYTES - 1) / GB_BYTES; } + +/** + * Convert displayed prune target GB to configured MiB. Round down so roundtrip GB -> MiB -> GB conversion is stable. + */ +static inline int64_t PruneGBtoMiB(int gb) { return gb * GB_BYTES / 1024 / 1024; } + /** Interface from Qt to configuration data structure for Bitcoin client. To Qt, the options are presented as a list with the different options laid out vertically. @@ -73,7 +84,8 @@ public: const QString& getOverriddenByCommandLine() { return strOverriddenByCommandLine; } /* Explicit setters */ - void SetPrune(bool prune, bool force = false); + void SetPruneEnabled(bool prune, bool force = false); + void SetPruneTargetGB(int prune_target_gb, bool force = false); /* Restart flag helper */ void setRestartRequired(bool fRequired); diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 07ffff0126..342c7cce31 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 6ad219ca2d..beca78a021 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 8b2533508d..aa9a7327ba 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,13 +36,17 @@ #include <config/bitcoin-config.h> #endif -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <QObject> #include <QString> class OptionsModel; +namespace interfaces { +class Node; +} // namespace interfaces + QT_BEGIN_NAMESPACE class QApplication; class QByteArray; diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index af2a1bb0e5..a497f58b16 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -1,16 +1,15 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/peertablemodel.h> -#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <interfaces/node.h> -#include <algorithm> +#include <utility> #include <QDebug> #include <QList> @@ -100,10 +99,9 @@ public: } }; -PeerTableModel::PeerTableModel(interfaces::Node& node, ClientModel *parent) : +PeerTableModel::PeerTableModel(interfaces::Node& node, QObject* parent) : QAbstractTableModel(parent), m_node(node), - clientModel(parent), timer(nullptr) { columns << tr("NodeId") << tr("Node/Service") << tr("Ping") << tr("Sent") << tr("Received") << tr("User Agent"); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index b3f5dd7dbe..cf45c5a08f 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -13,7 +13,6 @@ #include <QAbstractTableModel> #include <QStringList> -class ClientModel; class PeerTablePriv; namespace interfaces { @@ -51,7 +50,7 @@ class PeerTableModel : public QAbstractTableModel Q_OBJECT public: - explicit PeerTableModel(interfaces::Node& node, ClientModel *parent = nullptr); + explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); const CNodeCombinedStats *getNodeStats(int idx); int getRowByNodeId(NodeId nodeid); @@ -83,7 +82,6 @@ public Q_SLOTS: private: interfaces::Node& m_node; - ClientModel *clientModel; QStringList columns; std::unique_ptr<PeerTablePriv> priv; QTimer *timer; diff --git a/src/qt/platformstyle.cpp b/src/qt/platformstyle.cpp index 08d692e44c..c6b80fd340 100644 --- a/src/qt/platformstyle.cpp +++ b/src/qt/platformstyle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/platformstyle.h b/src/qt/platformstyle.h index 635aec4c93..53632e56e2 100644 --- a/src/qt/platformstyle.h +++ b/src/qt/platformstyle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index 2332d52b9a..c816e1f8ed 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qrimagewidget.h b/src/qt/qrimagewidget.h index 2a219ac101..345bb64092 100644 --- a/src/qt/qrimagewidget.h +++ b/src/qt/qrimagewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qvaluecombobox.h b/src/qt/qvaluecombobox.h index 8892071fba..5cca515079 100644 --- a/src/qt/qvaluecombobox.h +++ b/src/qt/qvaluecombobox.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index de453cf743..16597e4758 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index e492502002..b4fae7d78d 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <qt/bitcoinunits.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> +#include <qt/walletmodel.h> #include <QClipboard> #include <QPixmap> diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index a6e1a2af16..40e3d5ffa8 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -1,14 +1,16 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_RECEIVEREQUESTDIALOG_H #define BITCOIN_QT_RECEIVEREQUESTDIALOG_H -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <QDialog> +class WalletModel; + namespace Ui { class ReceiveRequestDialog; } diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 1611ec823c..7419297a96 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,12 +7,12 @@ #include <qt/bitcoinunits.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> +#include <qt/walletmodel.h> #include <clientversion.h> #include <streams.h> -#include <algorithm> - +#include <utility> RecentRequestsTableModel::RecentRequestsTableModel(WalletModel *parent) : QAbstractTableModel(parent), walletModel(parent) @@ -213,10 +213,10 @@ void RecentRequestsTableModel::updateDisplayUnit() updateAmountColumnTitle(); } -bool RecentRequestEntryLessThan::operator()(RecentRequestEntry &left, RecentRequestEntry &right) const +bool RecentRequestEntryLessThan::operator()(const RecentRequestEntry& left, const RecentRequestEntry& right) const { - RecentRequestEntry *pLeft = &left; - RecentRequestEntry *pRight = &right; + const RecentRequestEntry* pLeft = &left; + const RecentRequestEntry* pRight = &right; if (order == Qt::DescendingOrder) std::swap(pLeft, pRight); diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 130b709d46..5e7f6acdc8 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -1,16 +1,18 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_RECENTREQUESTSTABLEMODEL_H #define BITCOIN_QT_RECENTREQUESTSTABLEMODEL_H -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <QAbstractTableModel> #include <QStringList> #include <QDateTime> +class WalletModel; + class RecentRequestEntry { public: @@ -43,7 +45,7 @@ class RecentRequestEntryLessThan public: RecentRequestEntryLessThan(int nColumn, Qt::SortOrder fOrder): column(nColumn), order(fOrder) {} - bool operator()(RecentRequestEntry &left, RecentRequestEntry &right) const; + bool operator()(const RecentRequestEntry& left, const RecentRequestEntry& right) const; private: int column; diff --git a/src/qt/res/movies/makespinner.sh b/src/qt/res/movies/makespinner.sh index 3507837da9..4fa8dadf86 100755 --- a/src/qt/res/movies/makespinner.sh +++ b/src/qt/res/movies/makespinner.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 3dd64c5273..e1f783b0e5 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -905,12 +905,8 @@ void RPCConsole::on_lineEdit_returnPressed() cmdBeforeBrowsing = QString(); - WalletModel* wallet_model{nullptr}; #ifdef ENABLE_WALLET - const int wallet_index = ui->WalletSelector->currentIndex(); - if (wallet_index > 0) { - wallet_model = ui->WalletSelector->itemData(wallet_index).value<WalletModel*>(); - } + WalletModel* wallet_model = ui->WalletSelector->currentData().value<WalletModel*>(); if (m_last_wallet_model != wallet_model) { if (wallet_model) { @@ -1240,7 +1236,7 @@ void RPCConsole::unbanSelectedNode() QString strNode = nodes.at(i).data().toString(); CSubNet possibleSubnet; - LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); + LookupSubNet(strNode.toStdString(), possibleSubnet); if (possibleSubnet.IsValid() && m_node.unban(possibleSubnet)) { clientModel->getBanTableModel()->refresh(); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 8edcca684d..cc01aafb23 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,11 +21,13 @@ #include <chainparams.h> #include <interfaces/node.h> #include <key_io.h> -#include <wallet/coincontrol.h> -#include <ui_interface.h> -#include <txmempool.h> #include <policy/fees.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <wallet/coincontrol.h> #include <wallet/fees.h> +#include <wallet/psbtwallet.h> +#include <wallet/wallet.h> #include <QFontMetrics> #include <QScrollBar> @@ -186,6 +188,11 @@ void SendCoinsDialog::setModel(WalletModel *_model) // set default rbf checkbox state ui->optInRBF->setCheckState(Qt::Checked); + if (model->privateKeysDisabled()) { + ui->sendButton->setText(tr("Cr&eate Unsigned")); + ui->sendButton->setToolTip(tr("Creates a Partially Signed Bitcoin Transaction (PSBT) for use with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + } + // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() != 0) { @@ -305,9 +312,19 @@ void SendCoinsDialog::on_sendButton_clicked() formatted.append(recipientElement); } - QString questionString = tr("Are you sure you want to send?"); + QString questionString; + if (model->privateKeysDisabled()) { + questionString.append(tr("Do you want to draft this transaction?")); + } else { + questionString.append(tr("Are you sure you want to send?")); + } + questionString.append("<br /><span style='font-size:10pt;'>"); - questionString.append(tr("Please, review your transaction.")); + if (model->privateKeysDisabled()) { + questionString.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + } else { + questionString.append(tr("Please, review your transaction.")); + } questionString.append("</span>%1"); if(txFee > 0) @@ -358,8 +375,9 @@ void SendCoinsDialog::on_sendButton_clicked() } else { questionString = questionString.arg("<br /><br />" + formatted.at(0)); } - - SendConfirmationDialog confirmationDialog(tr("Confirm send coins"), questionString, informative_text, detailed_text, SEND_CONFIRM_DELAY, this); + const QString confirmation = model->privateKeysDisabled() ? tr("Confirm transaction proposal") : tr("Confirm send coins"); + const QString confirmButtonText = model->privateKeysDisabled() ? tr("Copy PSBT to clipboard") : tr("Send"); + SendConfirmationDialog confirmationDialog(confirmation, questionString, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); confirmationDialog.exec(); QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); @@ -369,17 +387,35 @@ void SendCoinsDialog::on_sendButton_clicked() return; } - // now send the prepared transaction - WalletModel::SendCoinsReturn sendStatus = model->sendCoins(currentTransaction); - // process sendStatus and on error generate message shown to user - processSendCoinsReturn(sendStatus); + bool send_failure = false; + if (model->privateKeysDisabled()) { + CMutableTransaction mtx = CMutableTransaction{*(currentTransaction.getWtx())}; + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + const TransactionError err = model->wallet().fillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + assert(!complete); + assert(err == TransactionError::OK); + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + } else { + // now send the prepared transaction + WalletModel::SendCoinsReturn sendStatus = model->sendCoins(currentTransaction); + // process sendStatus and on error generate message shown to user + processSendCoinsReturn(sendStatus); - if (sendStatus.status == WalletModel::OK) - { + if (sendStatus.status == WalletModel::OK) { + Q_EMIT coinsSent(currentTransaction.getWtx()->GetHash()); + } else { + send_failure = true; + } + } + if (!send_failure) { accept(); CoinControlDialog::coinControl()->UnSelectAll(); coinControlUpdateLabels(); - Q_EMIT coinsSent(currentTransaction.getWtx()->GetHash()); } fNewRecipientAllowed = true; } @@ -526,7 +562,12 @@ void SendCoinsDialog::setBalance(const interfaces::WalletBalances& balances) { if(model && model->getOptionsModel()) { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), balances.balance)); + CAmount balance = balances.balance; + if (model->privateKeysDisabled()) { + balance = balances.watch_only_balance; + ui->labelBalanceName->setText(tr("Watch-only balance:")); + } + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), balance)); } } @@ -611,6 +652,9 @@ void SendCoinsDialog::useAvailableBalance(SendCoinsEntry* entry) coin_control = *CoinControlDialog::coinControl(); } + // Include watch-only for wallets without private key + coin_control.fAllowWatchOnly = model->privateKeysDisabled(); + // Calculate available amount to send. CAmount amount = model->wallet().getAvailableBalance(coin_control); for (int i = 0; i < ui->entries->count(); ++i) { @@ -663,6 +707,8 @@ void SendCoinsDialog::updateCoinControlState(CCoinControl& ctrl) // Either custom fee will be used or if not selected, the confirmation target from dropdown box ctrl.m_confirm_target = getConfTargetForIndex(ui->confTargetSelector->currentIndex()); ctrl.m_signal_bip125_rbf = ui->optInRBF->isChecked(); + // Include watch-only for wallets without private key + ctrl.fAllowWatchOnly = model->privateKeysDisabled(); } void SendCoinsDialog::updateSmartFeeLabel() @@ -870,8 +916,8 @@ void SendCoinsDialog::coinControlUpdateLabels() } } -SendConfirmationDialog::SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text, const QString& detailed_text, int _secDelay, QWidget* parent) - : QMessageBox(parent), secDelay(_secDelay) +SendConfirmationDialog::SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text, const QString& detailed_text, int _secDelay, const QString& _confirmButtonText, QWidget* parent) + : QMessageBox(parent), secDelay(_secDelay), confirmButtonText(_confirmButtonText) { setIcon(QMessageBox::Question); setWindowTitle(title); // On macOS, the window title is ignored (as required by the macOS Guidelines). @@ -908,11 +954,11 @@ void SendConfirmationDialog::updateYesButton() if(secDelay > 0) { yesButton->setEnabled(false); - yesButton->setText(tr("Send") + " (" + QString::number(secDelay) + ")"); + yesButton->setText(confirmButtonText + " (" + QString::number(secDelay) + ")"); } else { yesButton->setEnabled(true); - yesButton->setText(tr("Send")); + yesButton->setText(confirmButtonText); } } diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index c6c1816877..86422c4030 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -108,7 +108,7 @@ class SendConfirmationDialog : public QMessageBox Q_OBJECT public: - SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, QWidget* parent = nullptr); + SendConfirmationDialog(const QString& title, const QString& text, const QString& informative_text = "", const QString& detailed_text = "", int secDelay = SEND_CONFIRM_DELAY, const QString& confirmText = "Send", QWidget* parent = nullptr); int exec(); private Q_SLOTS: @@ -119,6 +119,7 @@ private: QAbstractButton *yesButton; QTimer countDownTimer; int secDelay; + QString confirmButtonText; }; #endif // BITCOIN_QT_SENDCOINSDIALOG_H diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index be417655b4..444dc79a2e 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ #include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> +#include <qt/walletmodel.h> #include <QApplication> #include <QClipboard> @@ -36,7 +37,6 @@ SendCoinsEntry::SendCoinsEntry(const PlatformStyle *_platformStyle, QWidget *par if (platformStyle->getUseExtraSpacing()) ui->payToLayout->setSpacing(4); - ui->addAsLabel->setPlaceholderText(tr("Enter a label for this address to add it to your address book")); // normal bitcoin address field GUIUtil::setupAddressWidget(ui->payTo, this); diff --git a/src/qt/sendcoinsentry.h b/src/qt/sendcoinsentry.h index 42e2217130..254cc186e2 100644 --- a/src/qt/sendcoinsentry.h +++ b/src/qt/sendcoinsentry.h @@ -1,17 +1,21 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_SENDCOINSENTRY_H #define BITCOIN_QT_SENDCOINSENTRY_H -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <QStackedWidget> class WalletModel; class PlatformStyle; +namespace interfaces { +class Node; +} // namespace interfaces + namespace Ui { class SendCoinsEntry; } diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h new file mode 100644 index 0000000000..12279fab64 --- /dev/null +++ b/src/qt/sendcoinsrecipient.h @@ -0,0 +1,74 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_SENDCOINSRECIPIENT_H +#define BITCOIN_QT_SENDCOINSRECIPIENT_H + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <amount.h> +#include <serialize.h> + +#include <string> + +#include <QString> + +class SendCoinsRecipient +{ +public: + explicit SendCoinsRecipient() : amount(0), fSubtractFeeFromAmount(false), nVersion(SendCoinsRecipient::CURRENT_VERSION) { } + explicit SendCoinsRecipient(const QString &addr, const QString &_label, const CAmount& _amount, const QString &_message): + address(addr), label(_label), amount(_amount), message(_message), fSubtractFeeFromAmount(false), nVersion(SendCoinsRecipient::CURRENT_VERSION) {} + + // If from an unauthenticated payment request, this is used for storing + // the addresses, e.g. address-A<br />address-B<br />address-C. + // Info: As we don't need to process addresses in here when using + // payment requests, we can abuse it for displaying an address list. + // Todo: This is a hack, should be replaced with a cleaner solution! + QString address; + QString label; + CAmount amount; + // If from a payment request, this is used for storing the memo + QString message; + // Keep the payment request around as a serialized string to ensure + // load/store is lossless. + std::string sPaymentRequest; + // Empty if no authentication or invalid signature/cert/etc. + QString authenticatedMerchant; + + bool fSubtractFeeFromAmount; // memory only + + static const int CURRENT_VERSION = 1; + int nVersion; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + std::string sAddress = address.toStdString(); + std::string sLabel = label.toStdString(); + std::string sMessage = message.toStdString(); + std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); + + READWRITE(this->nVersion); + READWRITE(sAddress); + READWRITE(sLabel); + READWRITE(amount); + READWRITE(sMessage); + READWRITE(sPaymentRequest); + READWRITE(sAuthenticatedMerchant); + + if (ser_action.ForRead()) + { + address = QString::fromStdString(sAddress); + label = QString::fromStdString(sLabel); + message = QString::fromStdString(sMessage); + authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); + } + } +}; + +#endif // BITCOIN_QT_SENDCOINSRECIPIENT_H diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 1d0e1323bc..5f2836cc75 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,8 +35,6 @@ SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformS ui->verifyMessageButton_VM->setIcon(platformStyle->SingleColorIcon(":/icons/transaction_0")); ui->clearButton_VM->setIcon(platformStyle->SingleColorIcon(":/icons/remove")); - ui->signatureOut_SM->setPlaceholderText(tr("Click \"Sign Message\" to generate signature")); - GUIUtil::setupAddressWidget(ui->addressIn_SM, this); GUIUtil::setupAddressWidget(ui->addressIn_VM, this); @@ -136,7 +134,7 @@ void SignVerifyMessageDialog::on_signMessageButton_SM_clicked() } CKey key; - if (!model->wallet().getPrivKey(CKeyID(*pkhash), key)) + if (!model->wallet().getPrivKey(GetScriptForDestination(destination), CKeyID(*pkhash), key)) { ui->statusLabel_SM->setStyleSheet("QLabel { color: red; }"); ui->statusLabel_SM->setText(tr("Private key for the entered address is not available.")); diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 26c9fe7ad4..e19833019d 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 4fe440a679..0f082802cc 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,6 +1,10 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <qt/test/addressbooktests.h> #include <qt/test/util.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <interfaces/chain.h> #include <interfaces/node.h> @@ -51,11 +55,11 @@ void EditAddressAndSubmit( * In each case, verify the resulting state of the address book and optionally * the warning message presented to the user. */ -void TestAddAddressesToSendBook() +void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; - auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + wallet->SetupLegacyScriptPubKeyMan(); bool firstRun; wallet->LoadWallet(firstRun); @@ -101,10 +105,9 @@ void TestAddAddressesToSendBook() // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); - auto node = interfaces::MakeNode(); - OptionsModel optionsModel(*node); + OptionsModel optionsModel(node); AddWallet(wallet); - WalletModel walletModel(std::move(node->getWallets()[0]), *node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); RemoveWallet(wallet); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); @@ -150,5 +153,5 @@ void AddressBookTests::addressBookTests() return; } #endif - TestAddAddressesToSendBook(); + TestAddAddressesToSendBook(m_node); } diff --git a/src/qt/test/addressbooktests.h b/src/qt/test/addressbooktests.h index beeb9e76a9..5de89c7592 100644 --- a/src/qt/test/addressbooktests.h +++ b/src/qt/test/addressbooktests.h @@ -1,11 +1,23 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_QT_TEST_ADDRESSBOOKTESTS_H #define BITCOIN_QT_TEST_ADDRESSBOOKTESTS_H #include <QObject> #include <QTest> +namespace interfaces { +class Node; +} // namespace interfaces + class AddressBookTests : public QObject { +public: + AddressBookTests(interfaces::Node& node) : m_node(node) {} + interfaces::Node& m_node; + Q_OBJECT private Q_SLOTS: diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index e730c8f6d5..14a75b23f3 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,7 @@ #include <qt/networkstyle.h> #include <qt/rpcconsole.h> #include <shutdown.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <univalue.h> #include <validation.h> diff --git a/src/qt/test/apptests.h b/src/qt/test/apptests.h index 83bf56f1e4..d16c9fe487 100644 --- a/src/qt/test/apptests.h +++ b/src/qt/test/apptests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/compattests.cpp b/src/qt/test/compattests.cpp index cf86a5bc1e..c76dee5091 100644 --- a/src/qt/test/compattests.cpp +++ b/src/qt/test/compattests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 3c2ffa6c00..de1fbcb94c 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,7 +7,7 @@ #include <interfaces/node.h> #include <rpc/server.h> #include <qt/rpcconsole.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <univalue.h> #include <util/system.h> @@ -32,7 +32,6 @@ void RPCNestedTests::rpcNestedTests() // do some test setup // could be moved to a more generic place when we add more tests on QT level tableRPC.appendCommand("rpcNestedTest", &vRPCCommands[0]); - //mempool.setSanityCheck(1.0); TestingSetup test; @@ -41,7 +40,7 @@ void RPCNestedTests::rpcNestedTests() std::string result; std::string result2; std::string filtered; - auto node = interfaces::MakeNode(); + interfaces::Node* node = &m_node; RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo()[chain]", &filtered); //simple result filtering with path QVERIFY(result=="main"); QVERIFY(filtered == "getblockchaininfo()[chain]"); diff --git a/src/qt/test/rpcnestedtests.h b/src/qt/test/rpcnestedtests.h index 97143ff78a..0a00d1113a 100644 --- a/src/qt/test/rpcnestedtests.h +++ b/src/qt/test/rpcnestedtests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,16 @@ #include <QObject> #include <QTest> +namespace interfaces { +class Node; +} // namespace interfaces + class RPCNestedTests : public QObject { +public: + RPCNestedTests(interfaces::Node& node) : m_node(node) {} + interfaces::Node& m_node; + Q_OBJECT private Q_SLOTS: diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index f272627f96..aefdcd2716 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <qt/test/rpcnestedtests.h> #include <qt/test/uritests.h> #include <qt/test/compattests.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #ifdef ENABLE_WALLET #include <qt/test/addressbooktests.h> @@ -37,6 +37,8 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); #endif #endif +const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; + // This is all you need to run all the tests int main(int argc, char *argv[]) { @@ -50,7 +52,7 @@ int main(int argc, char *argv[]) BasicTestingSetup dummy{CBaseChainParams::REGTEST}; } - auto node = interfaces::MakeNode(); + std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(); bool fInvalid = false; @@ -76,7 +78,7 @@ int main(int argc, char *argv[]) if (QTest::qExec(&test1) != 0) { fInvalid = true; } - RPCNestedTests test3; + RPCNestedTests test3(*node); if (QTest::qExec(&test3) != 0) { fInvalid = true; } @@ -85,11 +87,11 @@ int main(int argc, char *argv[]) fInvalid = true; } #ifdef ENABLE_WALLET - WalletTests test5; + WalletTests test5(*node); if (QTest::qExec(&test5) != 0) { fInvalid = true; } - AddressBookTests test6; + AddressBookTests test6(*node); if (QTest::qExec(&test6) != 0) { fInvalid = true; } diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index ae2fb93bf7..e09f0ad77d 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <QApplication> #include <QMessageBox> #include <QPushButton> diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 377f07dcba..763847606a 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -1,3 +1,7 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_QT_TEST_UTIL_H #define BITCOIN_QT_TEST_UTIL_H diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index eea874c0d4..c1a0f63f73 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2015-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <qt/test/wallettests.h> #include <qt/test/util.h> @@ -13,7 +17,7 @@ #include <qt/transactionview.h> #include <qt/walletmodel.h> #include <key_io.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <validation.h> #include <wallet/wallet.h> #include <qt/overviewpage.h> @@ -126,21 +130,25 @@ void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, st // QT_QPA_PLATFORM=xcb src/qt/test/test_bitcoin-qt # Linux // QT_QPA_PLATFORM=windows src/qt/test/test_bitcoin-qt # Windows // QT_QPA_PLATFORM=cocoa src/qt/test/test_bitcoin-qt # macOS -void TestGUI() +void TestGUI(interfaces::Node& node) { // Set up wallet and chain with 105 blocks (5 mature blocks for spending). TestChain100Setup test; for (int i = 0; i < 5; ++i) { test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); } - auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + node.context()->connman = std::move(test.m_node.connman); + node.context()->mempool = std::move(test.m_node.mempool); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); { - LOCK(wallet->cs_wallet); + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + auto locked_chain = wallet->chain().lock(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); - wallet->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { auto locked_chain = wallet->chain().lock(); @@ -159,14 +167,23 @@ void TestGUI() std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); - auto node = interfaces::MakeNode(); - OptionsModel optionsModel(*node); + OptionsModel optionsModel(node); AddWallet(wallet); - WalletModel walletModel(std::move(node->getWallets().back()), *node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); RemoveWallet(wallet); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); + { + // Check balance in send dialog + QLabel* balanceLabel = sendCoinsDialog.findChild<QLabel*>("labelBalance"); + QString balanceText = balanceLabel->text(); + int unit = walletModel.getOptionsModel()->getDisplayUnit(); + CAmount balance = walletModel.wallet().getBalance(); + QString balanceComparison = BitcoinUnits::formatWithUnit(unit, balance, false, BitcoinUnits::separatorAlways); + QCOMPARE(balanceText, balanceComparison); + } + // Send two transactions, and verify they are added to transaction list. TransactionTableModel* transactionTableModel = walletModel.getTransactionTableModel(); QCOMPARE(transactionTableModel->rowCount({}), 105); @@ -260,5 +277,5 @@ void WalletTests::walletTests() return; } #endif - TestGUI(); + TestGUI(m_node); } diff --git a/src/qt/test/wallettests.h b/src/qt/test/wallettests.h index 342f7916c3..8ee40bf07f 100644 --- a/src/qt/test/wallettests.h +++ b/src/qt/test/wallettests.h @@ -1,11 +1,23 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_QT_TEST_WALLETTESTS_H #define BITCOIN_QT_TEST_WALLETTESTS_H #include <QObject> #include <QTest> +namespace interfaces { +class Node; +} // namespace interfaces + class WalletTests : public QObject { + public: + WalletTests(interfaces::Node& node) : m_node(node) {} + interfaces::Node& m_node; + Q_OBJECT private Q_SLOTS: diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 006007be63..757648f485 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index 48bd246b34..af5890ba24 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 318b0756c7..ece3a9cf48 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,11 +15,12 @@ #include <consensus/consensus.h> #include <interfaces/node.h> +#include <interfaces/wallet.h> #include <key_io.h> -#include <validation.h> +#include <policy/policy.h> #include <script/script.h> #include <util/system.h> -#include <policy/policy.h> +#include <validation.h> #include <wallet/ismine.h> #include <stdint.h> diff --git a/src/qt/transactiondescdialog.h b/src/qt/transactiondescdialog.h index 8fd3f3166a..74e34cde87 100644 --- a/src/qt/transactiondescdialog.h +++ b/src/qt/transactiondescdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 08ba030d65..a32d218fc9 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index fed55577ca..64e9c856db 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index cbc4ab49f5..3c638fb358 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 79347c371f..eca5656077 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 095c98d26f..3101fb01c3 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,8 +10,6 @@ #include <qt/forms/ui_helpmessagedialog.h> -#include <qt/bitcoingui.h> - #include <clientversion.h> #include <init.h> #include <util/system.h> @@ -21,9 +19,10 @@ #include <QCloseEvent> #include <QLabel> +#include <QMainWindow> #include <QRegExp> -#include <QTextTable> #include <QTextCursor> +#include <QTextTable> #include <QVBoxLayout> /** "Help message" or "About" dialog box */ @@ -34,14 +33,6 @@ HelpMessageDialog::HelpMessageDialog(interfaces::Node& node, QWidget *parent, bo ui->setupUi(this); QString version = QString{PACKAGE_NAME} + " " + tr("version") + " " + QString::fromStdString(FormatFullVersion()); - /* On x86 add a bit specifier to the version so that users can distinguish between - * 32 and 64 bit builds. On other architectures, 32/64 bit may be more ambiguous. - */ -#if defined(__x86_64__) - version += " " + tr("(%1-bit)").arg(64); -#elif defined(__i386__ ) - version += " " + tr("(%1-bit)").arg(32); -#endif if (about) { @@ -152,10 +143,9 @@ ShutdownWindow::ShutdownWindow(QWidget *parent, Qt::WindowFlags f): setLayout(layout); } -QWidget *ShutdownWindow::showShutdownWindow(BitcoinGUI *window) +QWidget* ShutdownWindow::showShutdownWindow(QMainWindow* window) { - if (!window) - return nullptr; + assert(window != nullptr); // Show a simple window indicating shutdown status QWidget *shutdownWindow = new ShutdownWindow(); diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index f1cedff282..833b86fd3e 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -6,9 +6,11 @@ #define BITCOIN_QT_UTILITYDIALOG_H #include <QDialog> -#include <QObject> +#include <QWidget> -class BitcoinGUI; +QT_BEGIN_NAMESPACE +class QMainWindow; +QT_END_NAMESPACE namespace interfaces { class Node; @@ -46,7 +48,7 @@ class ShutdownWindow : public QWidget public: explicit ShutdownWindow(QWidget *parent=nullptr, Qt::WindowFlags f=Qt::Widget); - static QWidget *showShutdownWindow(BitcoinGUI *window); + static QWidget* showShutdownWindow(QMainWindow* window); protected: void closeEvent(QCloseEvent *event); diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 4c253f8ddd..7413a1f09e 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -2,17 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <qt/walletcontroller.h> + #include <qt/askpassphrasedialog.h> #include <qt/createwalletdialog.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> -#include <qt/walletcontroller.h> - -#include <wallet/wallet.h> +#include <qt/walletmodel.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <util/string.h> +#include <wallet/wallet.h> #include <algorithm> diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index e50dd5c7eb..956245775e 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_WALLETCONTROLLER_H #define BITCOIN_QT_WALLETCONTROLLER_H -#include <qt/walletmodel.h> +#include <qt/sendcoinsrecipient.h> #include <support/allocators/secure.h> #include <sync.h> @@ -23,10 +23,12 @@ class OptionsModel; class PlatformStyle; +class WalletModel; namespace interfaces { class Handler; class Node; +class Wallet; } // namespace interfaces class AskPassphraseDialog; diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index d7f0617315..656fffd067 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,14 +39,13 @@ void WalletFrame::setClientModel(ClientModel *_clientModel) this->clientModel = _clientModel; } -void WalletFrame::addWallet(WalletModel *walletModel) +bool WalletFrame::addWallet(WalletModel *walletModel) { - if (!gui || !clientModel || !walletModel) return; + if (!gui || !clientModel || !walletModel) return false; - if (mapWalletViews.count(walletModel) > 0) return; + if (mapWalletViews.count(walletModel) > 0) return false; WalletView *walletView = new WalletView(platformStyle, this); - walletView->setBitcoinGUI(gui); walletView->setClientModel(clientModel); walletView->setWalletModel(walletModel); walletView->showOutOfSyncWarning(bOutOfSync); @@ -61,12 +60,17 @@ void WalletFrame::addWallet(WalletModel *walletModel) walletStack->addWidget(walletView); mapWalletViews[walletModel] = walletView; - // Ensure a walletView is able to show the main window - connect(walletView, &WalletView::showNormalIfMinimized, [this]{ - gui->showNormalIfMinimized(); + connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); + connect(walletView, &WalletView::transactionClicked, gui, &BitcoinGUI::gotoHistoryPage); + connect(walletView, &WalletView::coinsSent, gui, &BitcoinGUI::gotoHistoryPage); + connect(walletView, &WalletView::message, [this](const QString& title, const QString& message, unsigned int style) { + gui->message(title, message, style); }); + connect(walletView, &WalletView::encryptionStatusChanged, gui, &BitcoinGUI::updateWalletStatus); + connect(walletView, &WalletView::incomingTransaction, gui, &BitcoinGUI::incomingTransaction); + connect(walletView, &WalletView::hdEnabledStatusChanged, gui, &BitcoinGUI::updateWalletStatus); - connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); + return true; } void WalletFrame::setCurrentWallet(WalletModel* wallet_model) diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 156653f47d..20fad08b0e 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -36,7 +36,7 @@ public: void setClientModel(ClientModel *clientModel); - void addWallet(WalletModel *walletModel); + bool addWallet(WalletModel *walletModel); void setCurrentWallet(WalletModel* wallet_model); void removeWallet(WalletModel* wallet_model); void removeAllWallets(); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 33801d3907..6c3a06f3a2 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -10,6 +10,7 @@ #include <qt/addresstablemodel.h> #include <qt/guiconstants.h> +#include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/paymentserver.h> #include <qt/recentrequeststablemodel.h> @@ -183,7 +184,7 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact std::string strFailReason; auto& newTx = transaction.getWtx(); - newTx = m_wallet->createTransaction(vecSend, coinControl, true /* sign */, nChangePosRet, nFeeRequired, strFailReason); + newTx = m_wallet->createTransaction(vecSend, coinControl, !privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired, strFailReason); transaction.setTransactionFee(nFeeRequired); if (fSubtractFeeFromAmount && newTx) transaction.reassignAmounts(nChangePosRet); @@ -487,8 +488,10 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return false; } + const bool create_psbt = privateKeysDisabled(); + // allow a user based fee verification - QString questionString = tr("Do you want to increase the fee?"); + QString questionString = create_psbt ? tr("Do you want to draft a transaction with fee increase?") : tr("Do you want to increase the fee?"); questionString.append("<br />"); questionString.append("<table style=\"text-align: left;\">"); questionString.append("<tr><td>"); @@ -519,6 +522,23 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return false; } + // Short-circuit if we are returning a bumped transaction PSBT to clipboard + if (create_psbt) { + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + const TransactionError err = wallet().fillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + if (err != TransactionError::OK || complete) { + QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't draft transaction.")); + return false; + } + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + return true; + } + // sign bumped transaction if (!m_wallet->signBumpTransaction(mtx)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't sign transaction.")); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index d21dec118a..8087356f5e 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -9,9 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> #include <key.h> -#include <serialize.h> #include <script/standard.h> #include <qt/walletmodeltransaction.h> @@ -29,6 +27,7 @@ class AddressTableModel; class OptionsModel; class PlatformStyle; class RecentRequestsTableModel; +class SendCoinsRecipient; class TransactionTableModel; class WalletModelTransaction; @@ -47,61 +46,6 @@ QT_BEGIN_NAMESPACE class QTimer; QT_END_NAMESPACE -class SendCoinsRecipient -{ -public: - explicit SendCoinsRecipient() : amount(0), fSubtractFeeFromAmount(false), nVersion(SendCoinsRecipient::CURRENT_VERSION) { } - explicit SendCoinsRecipient(const QString &addr, const QString &_label, const CAmount& _amount, const QString &_message): - address(addr), label(_label), amount(_amount), message(_message), fSubtractFeeFromAmount(false), nVersion(SendCoinsRecipient::CURRENT_VERSION) {} - - // If from an unauthenticated payment request, this is used for storing - // the addresses, e.g. address-A<br />address-B<br />address-C. - // Info: As we don't need to process addresses in here when using - // payment requests, we can abuse it for displaying an address list. - // Todo: This is a hack, should be replaced with a cleaner solution! - QString address; - QString label; - CAmount amount; - // If from a payment request, this is used for storing the memo - QString message; - // If building with BIP70 is disabled, keep the payment request around as - // serialized string to ensure load/store is lossless - std::string sPaymentRequest; - // Empty if no authentication or invalid signature/cert/etc. - QString authenticatedMerchant; - - bool fSubtractFeeFromAmount; // memory only - - static const int CURRENT_VERSION = 1; - int nVersion; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - std::string sAddress = address.toStdString(); - std::string sLabel = label.toStdString(); - std::string sMessage = message.toStdString(); - std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); - - READWRITE(this->nVersion); - READWRITE(sAddress); - READWRITE(sLabel); - READWRITE(amount); - READWRITE(sMessage); - READWRITE(sPaymentRequest); - READWRITE(sAuthenticatedMerchant); - - if (ser_action.ForRead()) - { - address = QString::fromStdString(sAddress); - label = QString::fromStdString(sLabel); - message = QString::fromStdString(sMessage); - authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); - } - } -}; - /** Interface to Bitcoin wallet from Qt view code. */ class WalletModel : public QObject { diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index eba95bd27c..25172e774c 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index 242ba13897..f9a95362c8 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -1,11 +1,12 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_WALLETMODELTRANSACTION_H #define BITCOIN_QT_WALLETMODELTRANSACTION_H -#include <qt/walletmodel.h> +#include <primitives/transaction.h> +#include <qt/sendcoinsrecipient.h> #include <amount.h> diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 8652827b59..bdcb82e06b 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,6 @@ #include <qt/addressbookpage.h> #include <qt/askpassphrasedialog.h> -#include <qt/bitcoingui.h> #include <qt/clientmodel.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -65,11 +64,13 @@ WalletView::WalletView(const PlatformStyle *_platformStyle, QWidget *parent): addWidget(receiveCoinsPage); addWidget(sendCoinsPage); + connect(overviewPage, &OverviewPage::transactionClicked, this, &WalletView::transactionClicked); // Clicking on a transaction on the overview pre-selects the transaction on the transaction history page connect(overviewPage, &OverviewPage::transactionClicked, transactionView, static_cast<void (TransactionView::*)(const QModelIndex&)>(&TransactionView::focusTransaction)); connect(overviewPage, &OverviewPage::outOfSyncWarningClicked, this, &WalletView::requestedSyncWarningInfo); + connect(sendCoinsPage, &SendCoinsDialog::coinsSent, this, &WalletView::coinsSent); // Highlight transaction after send connect(sendCoinsPage, &SendCoinsDialog::coinsSent, transactionView, static_cast<void (TransactionView::*)(const uint256&)>(&TransactionView::focusTransaction)); @@ -86,32 +87,6 @@ WalletView::~WalletView() { } -void WalletView::setBitcoinGUI(BitcoinGUI *gui) -{ - if (gui) - { - // Clicking on a transaction on the overview page simply sends you to transaction history page - connect(overviewPage, &OverviewPage::transactionClicked, gui, &BitcoinGUI::gotoHistoryPage); - - // Navigate to transaction history page after send - connect(sendCoinsPage, &SendCoinsDialog::coinsSent, gui, &BitcoinGUI::gotoHistoryPage); - - // Receive and report messages - connect(this, &WalletView::message, [gui](const QString &title, const QString &message, unsigned int style) { - gui->message(title, message, style); - }); - - // Pass through encryption status changed signals - connect(this, &WalletView::encryptionStatusChanged, gui, &BitcoinGUI::updateWalletStatus); - - // Pass through transaction notifications - connect(this, &WalletView::incomingTransaction, gui, &BitcoinGUI::incomingTransaction); - - // Connect HD enabled state signal - connect(this, &WalletView::hdEnabledStatusChanged, gui, &BitcoinGUI::updateWalletStatus); - } -} - void WalletView::setClientModel(ClientModel *_clientModel) { this->clientModel = _clientModel; diff --git a/src/qt/walletview.h b/src/qt/walletview.h index e29c4c52f5..86e46348be 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,6 @@ #include <QStackedWidget> -class BitcoinGUI; class ClientModel; class OverviewPage; class PlatformStyle; @@ -39,7 +38,6 @@ public: explicit WalletView(const PlatformStyle *platformStyle, QWidget *parent); ~WalletView(); - void setBitcoinGUI(BitcoinGUI *gui); /** Set the client model. The client model represents the part of the core that communicates with the P2P network, and is wallet-agnostic. */ @@ -115,8 +113,8 @@ public Q_SLOTS: void requestedSyncWarningInfo(); Q_SIGNALS: - /** Signal that we want to show the main window */ - void showNormalIfMinimized(); + void transactionClicked(); + void coinsSent(); /** Fired when a message should be reported to the user */ void message(const QString &title, const QString &message, unsigned int style); /** Encryption status of wallet changed */ diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index c6eb133cbd..386d593eea 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.cpp b/src/random.cpp index 48d20d7d72..f0082cf3e0 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,23 +1,27 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <random.h> +#include <compat/cpuid.h> +#include <crypto/sha256.h> #include <crypto/sha512.h> #include <support/cleanse.h> #ifdef WIN32 #include <compat.h> // for Windows API #include <wincrypt.h> #endif -#include <logging.h> // for LogPrint() -#include <sync.h> // for WAIT_LOCK +#include <logging.h> // for LogPrintf() +#include <sync.h> // for Mutex #include <util/time.h> // for GetTime() #include <stdlib.h> #include <thread> +#include <randomenv.h> + #include <support/allocators/secure.h> #ifndef WIN32 @@ -40,15 +44,6 @@ #include <sys/sysctl.h> #endif - -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -#include <cpuid.h> -#endif - -#include <openssl/err.h> -#include <openssl/rand.h> -#include <openssl/conf.h> - [[noreturn]] static void RandFailure() { LogPrintf("Failed to read randomness, aborting\n"); @@ -75,7 +70,7 @@ static inline int64_t GetPerformanceCounter() noexcept #endif } -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#ifdef HAVE_GETCPUID static bool g_rdrand_supported = false; static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; @@ -86,15 +81,6 @@ static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND" #ifdef bit_RDSEED static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); #endif -static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ - // We can't use __get_cpuid as it doesn't support subleafs. -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} static void InitHardwareRand() { @@ -193,7 +179,7 @@ static uint64_t GetRdSeed() noexcept /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). * Slower sources should probably be invoked separately, and/or only from - * RandAddSeedSleep (which is called during idle background operation). + * RandAddPeriodic (which is called once a minute). */ static void InitHardwareRand() {} static void ReportHardwareRand() {} @@ -263,44 +249,6 @@ static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA51 memory_cleanse(buffer, sizeof(buffer)); } -static void RandAddSeedPerfmon(CSHA512& hasher) -{ -#ifdef WIN32 - // Don't need this on Linux, OpenSSL automatically uses /dev/urandom - // Seed with the entire set of perfmon data - - // This can take up to 2 seconds, so only do it every 10 minutes - static int64_t nLastPerfmon; - if (GetTime() < nLastPerfmon + 10 * 60) - return; - nLastPerfmon = GetTime(); - - std::vector<unsigned char> vData(250000, 0); - long ret = 0; - unsigned long nSize = 0; - const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data - while (true) { - nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); - if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) - break; - vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially - } - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) { - hasher.Write(vData.data(), nSize); - memory_cleanse(vData.data(), nSize); - } else { - // Performance data is only a best-effort attempt at improving the - // situation when the OS randomness (and other sources) aren't - // adequate. As a result, failure to read it is isn't considered critical, - // so we don't call RandFailure(). - // TODO: Add logging when the logger is made functional before global - // constructors have been invoked. - } -#endif -} - #ifndef WIN32 /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. @@ -396,8 +344,6 @@ void GetOSRand(unsigned char *ent32) #endif } -void LockingCallbackOpenSSL(int mode, int i, const char* file, int line); - namespace { class RNGState { @@ -413,31 +359,47 @@ class RNGState { unsigned char m_state[32] GUARDED_BY(m_mutex) = {0}; uint64_t m_counter GUARDED_BY(m_mutex) = 0; bool m_strongly_seeded GUARDED_BY(m_mutex) = false; - std::unique_ptr<Mutex[]> m_mutex_openssl; + + Mutex m_events_mutex; + CSHA256 m_events_hasher GUARDED_BY(m_events_mutex); public: RNGState() noexcept { InitHardwareRand(); + } + + ~RNGState() + { + } - // Init OpenSSL library multithreading support - m_mutex_openssl.reset(new Mutex[CRYPTO_num_locks()]); - CRYPTO_set_locking_callback(LockingCallbackOpenSSL); + void AddEvent(uint32_t event_info) noexcept + { + LOCK(m_events_mutex); - // OpenSSL can optionally load a config file which lists optional loadable modules and engines. - // We don't use them so we don't require the config. However some of our libs may call functions - // which attempt to load the config file, possibly resulting in an exit() or crash if it is missing - // or corrupt. Explicitly tell OpenSSL not to try to load the file. The result for our libs will be - // that the config appears to have been loaded and there are no modules/engines available. - OPENSSL_no_config(); + m_events_hasher.Write((const unsigned char *)&event_info, sizeof(event_info)); + // Get the low four bytes of the performance counter. This translates to roughly the + // subsecond part. + uint32_t perfcounter = (GetPerformanceCounter() & 0xffffffff); + m_events_hasher.Write((const unsigned char*)&perfcounter, sizeof(perfcounter)); } - ~RNGState() + /** + * Feed (the hash of) all events added through AddEvent() to hasher. + */ + void SeedEvents(CSHA512& hasher) noexcept { - // Securely erase the memory used by the OpenSSL PRNG - RAND_cleanup(); - // Shutdown OpenSSL library multithreading support - CRYPTO_set_locking_callback(nullptr); + // We use only SHA256 for the events hashing to get the ASM speedups we have for SHA256, + // since we want it to be fast as network peers may be able to trigger it repeatedly. + LOCK(m_events_mutex); + + unsigned char events_hash[32]; + m_events_hasher.Finalize(events_hash); + hasher.Write(events_hash, 32); + + // Re-initialize the hasher with the finalized state to use later. + m_events_hasher.Reset(); + m_events_hasher.Write(events_hash, 32); } /** Extract up to 32 bytes of entropy from the RNG state, mixing in new entropy from hasher. @@ -473,8 +435,6 @@ public: memory_cleanse(buf, 64); return ret; } - - Mutex& GetOpenSSLMutex(int i) { return m_mutex_openssl[i]; } }; RNGState& GetRNGState() noexcept @@ -486,30 +446,9 @@ RNGState& GetRNGState() noexcept } } -void LockingCallbackOpenSSL(int mode, int i, const char* file, int line) NO_THREAD_SAFETY_ANALYSIS -{ - RNGState& rng = GetRNGState(); - - if (mode & CRYPTO_LOCK) { - rng.GetOpenSSLMutex(i).lock(); - } else { - rng.GetOpenSSLMutex(i).unlock(); - } -} - /* A note on the use of noexcept in the seeding functions below: * - * None of the RNG code should ever throw any exception, with the sole exception - * of MilliSleep in SeedSleep, which can (and does) support interruptions which - * cause a boost::thread_interrupted to be thrown. - * - * This means that SeedSleep, and all functions that invoke it are throwing. - * However, we know that GetRandBytes() and GetStrongRandBytes() never trigger - * this sleeping logic, so they are noexcept. The same is true for all the - * GetRand*() functions that use GetRandBytes() indirectly. - * - * TODO: After moving away from interruptible boost-based thread management, - * everything can become noexcept here. + * None of the RNG code should ever throw any exception. */ static void SeedTimestamp(CSHA512& hasher) noexcept @@ -533,7 +472,7 @@ static void SeedFast(CSHA512& hasher) noexcept SeedTimestamp(hasher); } -static void SeedSlow(CSHA512& hasher) noexcept +static void SeedSlow(CSHA512& hasher, RNGState& rng) noexcept { unsigned char buffer[32]; @@ -544,9 +483,8 @@ static void SeedSlow(CSHA512& hasher) noexcept GetOSRand(buffer); hasher.Write(buffer, sizeof(buffer)); - // OpenSSL RNG (for now) - RAND_bytes(buffer, sizeof(buffer)); - hasher.Write(buffer, sizeof(buffer)); + // Add the events hasher into the mix + rng.SeedEvents(hasher); // High-precision timestamp. // @@ -556,22 +494,16 @@ static void SeedSlow(CSHA512& hasher) noexcept } /** Extract entropy from rng, strengthen it, and feed it into hasher. */ -static void SeedStrengthen(CSHA512& hasher, RNGState& rng) noexcept +static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noexcept { - static std::atomic<int64_t> last_strengthen{0}; - int64_t last_time = last_strengthen.load(); - int64_t current_time = GetTimeMicros(); - if (current_time > last_time + 60000000) { // Only run once a minute - // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. - unsigned char strengthen_seed[32]; - rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); - // Strengthen it for 10ms (100ms on first run), and feed it into hasher. - Strengthen(strengthen_seed, last_time == 0 ? 100000 : 10000, hasher); - last_strengthen = current_time; - } + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen the seed, and feed it into hasher. + Strengthen(strengthen_seed, microseconds, hasher); } -static void SeedSleep(CSHA512& hasher, RNGState& rng) +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) noexcept { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -579,17 +511,16 @@ static void SeedSleep(CSHA512& hasher, RNGState& rng) // High-precision timestamp SeedTimestamp(hasher); - // Sleep for 1ms - MilliSleep(1); + // Add the events hasher into the mix + rng.SeedEvents(hasher); - // High-precision timestamp after sleeping (as we commit to both the time before and after, this measures the delay) - SeedTimestamp(hasher); + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); + LogPrint(BCLog::RAND, "Feeding %i bytes of dynamic environment data into RNG\n", hasher.Size() - old_size); - // Windows performance monitor data (once every 10 minutes) - RandAddSeedPerfmon(hasher); - - // Strengthen every minute - SeedStrengthen(hasher, rng); + // Strengthen for 10 ms + SeedStrengthen(hasher, rng, 10000); } static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept @@ -598,22 +529,27 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept SeedHardwareSlow(hasher); // Everything that the 'slow' seeder includes. - SeedSlow(hasher); + SeedSlow(hasher, rng); + + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); - // Windows performance monitor data. - RandAddSeedPerfmon(hasher); + // Static environment data + RandAddStaticEnv(hasher); + LogPrint(BCLog::RAND, "Feeding %i bytes of environment data into RNG\n", hasher.Size() - old_size); - // Strengthen - SeedStrengthen(hasher, rng); + // Strengthen for 100 ms + SeedStrengthen(hasher, rng, 100000); } enum class RNGLevel { FAST, //!< Automatically called by GetRandBytes SLOW, //!< Automatically called by GetStrongRandBytes - SLEEP, //!< Called by RandAddSeedSleep() + PERIODIC, //!< Called by RandAddPeriodic() }; -static void ProcRand(unsigned char* out, int num, RNGLevel level) +static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept { // Make sure the RNG is initialized first (as all Seed* function possibly need hwrand to be available). RNGState& rng = GetRNGState(); @@ -626,10 +562,10 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedFast(hasher); break; case RNGLevel::SLOW: - SeedSlow(hasher); + SeedSlow(hasher, rng); break; - case RNGLevel::SLEEP: - SeedSleep(hasher, rng); + case RNGLevel::PERIODIC: + SeedPeriodic(hasher, rng); break; } @@ -640,19 +576,12 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) SeedStartup(startup_hasher, rng); rng.MixExtract(out, num, std::move(startup_hasher), true); } - - // For anything but the 'fast' level, feed the resulting RNG output (after an additional hashing step) back into OpenSSL. - if (level != RNGLevel::FAST) { - unsigned char buf[64]; - CSHA512().Write(out, num).Finalize(buf); - RAND_add(buf, sizeof(buf), num); - memory_cleanse(buf, 64); - } } void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } -void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } +void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } +void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } bool g_mock_deterministic_tests{false}; @@ -716,7 +645,7 @@ bool Random_SanityCheck() uint64_t start = GetPerformanceCounter(); /* This does not measure the quality of randomness, but it does test that - * OSRandom() overwrites all 32 bytes of the output given a maximum + * GetOSRand() overwrites all 32 bytes of the output given a maximum * number of tries. */ static const ssize_t MAX_TRIES = 1024; diff --git a/src/random.h b/src/random.h index 22801ec155..518a5cd3e3 100644 --- a/src/random.h +++ b/src/random.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,24 +35,22 @@ * that fast seeding includes, but additionally: * - OS entropy (/dev/urandom, getrandom(), ...). The application will terminate if * this entropy source fails. - * - Bytes from OpenSSL's RNG (which itself may be seeded from various sources) * - Another high-precision timestamp (indirectly committing to a benchmark of all the * previous sources). * These entropy sources are slower, but designed to make sure the RNG state contains * fresh data that is unpredictable to attackers. * - * - RandAddSeedSleep() seeds everything that fast seeding includes, but additionally: - * - A high-precision timestamp before and after sleeping 1ms. - * - (On Windows) Once every 10 minutes, performance monitoring data from the OS. - - - Once every minute, strengthen the entropy for 10 ms using repeated SHA512. - * These just exploit the fact the system is idle to improve the quality of the RNG - * slightly. + * - RandAddPeriodic() seeds everything that fast seeding includes, but additionally: + * - A high-precision timestamp + * - Dynamic environment data (performance monitoring, ...) + * - Strengthen the entropy for 10 ms using repeated SHA512. + * This is run once every minute. * * On first use of the RNG (regardless of what function is called first), all entropy * sources used in the 'slow' seeder are included, but also: * - 256 bits from the hardware RNG (rdseed or rdrand) when available. - * - (On Windows) Performance monitoring data from the OS. - * - (On Windows) Through OpenSSL, the screen contents. + * - Dynamic environment data (performance monitoring, ...) + * - Static environment data * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and @@ -85,11 +83,19 @@ uint256 GetRandHash() noexcept; void GetStrongRandBytes(unsigned char* buf, int num) noexcept; /** - * Sleep for 1ms, gather entropy from various sources, and feed them to the PRNG state. + * Gather entropy from various expensive sources, and feed them to the PRNG state. * * Thread-safe. */ -void RandAddSeedSleep(); +void RandAddPeriodic() noexcept; + +/** + * Gathers entropy from the low bits of the time at which events occur. Should + * be called with a uint32_t describing the event at the time an event occurs. + * + * Thread-safe. + */ +void RandAddEvent(const uint32_t event_info) noexcept; /** * Fast randomness source. This is seeded once with secure random data, but @@ -166,6 +172,7 @@ public: /** Generate a random integer in the range [0..range). */ uint64_t randrange(uint64_t range) noexcept { + assert(range); --range; int bits = CountBits(range); while (true) { diff --git a/src/randomenv.cpp b/src/randomenv.cpp new file mode 100644 index 0000000000..6992c720ff --- /dev/null +++ b/src/randomenv.cpp @@ -0,0 +1,518 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <randomenv.h> + +#include <clientversion.h> +#include <compat/cpuid.h> +#include <crypto/sha512.h> +#include <support/cleanse.h> +#include <util/time.h> // for GetTime() +#ifdef WIN32 +#include <compat.h> // for Windows API +#endif + +#include <algorithm> +#include <atomic> +#include <chrono> +#include <climits> +#include <thread> +#include <vector> + +#include <stdint.h> +#include <string.h> +#ifndef WIN32 +#include <sys/types.h> // must go before a number of other headers +#include <fcntl.h> +#include <netinet/in.h> +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/utsname.h> +#include <unistd.h> +#endif +#ifdef __MACH__ +#include <mach/clock.h> +#include <mach/mach.h> +#include <mach/mach_time.h> +#endif +#if HAVE_DECL_GETIFADDRS +#include <ifaddrs.h> +#endif +#if HAVE_SYSCTL +#include <sys/sysctl.h> +#if HAVE_VM_VM_PARAM_H +#include <vm/vm_param.h> +#endif +#if HAVE_SYS_RESOURCES_H +#include <sys/resources.h> +#endif +#if HAVE_SYS_VMMETER_H +#include <sys/vmmeter.h> +#endif +#endif +#ifdef __linux__ +#include <sys/auxv.h> +#endif + +//! Necessary on some platforms +extern char** environ; + +namespace { + +void RandAddSeedPerfmon(CSHA512& hasher) +{ +#ifdef WIN32 + // Seed with the entire set of perfmon data + + // This can take up to 2 seconds, so only do it every 10 minutes + static std::atomic<std::chrono::seconds> last_perfmon{std::chrono::seconds{0}}; + auto last_time = last_perfmon.load(); + auto current_time = GetTime<std::chrono::seconds>(); + if (current_time < last_time + std::chrono::minutes{10}) return; + last_perfmon = current_time; + + std::vector<unsigned char> vData(250000, 0); + long ret = 0; + unsigned long nSize = 0; + const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data + while (true) { + nSize = vData.size(); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); + if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) + break; + vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially + } + RegCloseKey(HKEY_PERFORMANCE_DATA); + if (ret == ERROR_SUCCESS) { + hasher.Write(vData.data(), nSize); + memory_cleanse(vData.data(), nSize); + } else { + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it is isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. + } +#endif +} + +/** Helper to easily feed data into a CSHA512. + * + * Note that this does not serialize the passed object (like stream.h's << operators do). + * Its raw memory representation is used directly. + */ +template<typename T> +CSHA512& operator<<(CSHA512& hasher, const T& data) { + static_assert(!std::is_same<typename std::decay<T>::type, char*>::value, "Calling operator<<(CSHA512, char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, unsigned char*>::value, "Calling operator<<(CSHA512, unsigned char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const char*>::value, "Calling operator<<(CSHA512, const char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const unsigned char*>::value, "Calling operator<<(CSHA512, const unsigned char*) is probably not what you want"); + hasher.Write((const unsigned char*)&data, sizeof(data)); + return hasher; +} + +#ifndef WIN32 +void AddSockaddr(CSHA512& hasher, const struct sockaddr *addr) +{ + if (addr == nullptr) return; + switch (addr->sa_family) { + case AF_INET: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in)); + break; + case AF_INET6: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in6)); + break; + default: + hasher.Write((const unsigned char*)&addr->sa_family, sizeof(addr->sa_family)); + } +} + +void AddFile(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + int f = open(path, O_RDONLY); + size_t total = 0; + if (f != -1) { + unsigned char fbuf[4096]; + int n; + hasher.Write((const unsigned char*)&f, sizeof(f)); + if (fstat(f, &sb) == 0) hasher << sb; + do { + n = read(f, fbuf, sizeof(fbuf)); + if (n > 0) hasher.Write(fbuf, n); + total += n; + /* not bothering with EINTR handling. */ + } while (n == sizeof(fbuf) && total < 1048576); // Read only the first 1 Mbyte + close(f); + } +} + +void AddPath(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + if (stat(path, &sb) == 0) { + hasher.Write((const unsigned char*)path, strlen(path) + 1); + hasher << sb; + } +} +#endif + +#if HAVE_SYSCTL +template<int... S> +void AddSysctl(CSHA512& hasher) +{ + int CTL[sizeof...(S)] = {S...}; + unsigned char buffer[65536]; + size_t siz = 65536; + int ret = sysctl(CTL, sizeof...(S), buffer, &siz, nullptr, 0); + if (ret == 0 || (ret == -1 && errno == ENOMEM)) { + hasher << sizeof(CTL); + hasher.Write((const unsigned char*)CTL, sizeof(CTL)); + if (siz > sizeof(buffer)) siz = sizeof(buffer); + hasher << siz; + hasher.Write(buffer, siz); + } +} +#endif + +#ifdef HAVE_GETCPUID +void inline AddCPUID(CSHA512& hasher, uint32_t leaf, uint32_t subleaf, uint32_t& ax, uint32_t& bx, uint32_t& cx, uint32_t& dx) +{ + GetCPUID(leaf, subleaf, ax, bx, cx, dx); + hasher << leaf << subleaf << ax << bx << cx << dx; +} + +void AddAllCPUID(CSHA512& hasher) +{ + uint32_t ax, bx, cx, dx; + // Iterate over all standard leaves + AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax + uint32_t max = ax; + for (uint32_t leaf = 1; leaf <= max && leaf <= 0xFF; ++leaf) { + uint32_t maxsub = 0; + for (uint32_t subleaf = 0; subleaf <= 0xFF; ++subleaf) { + AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); + // Iterate subleafs for leaf values 4, 7, 11, 13 + if (leaf == 4) { + if ((ax & 0x1f) == 0) break; + } else if (leaf == 7) { + if (subleaf == 0) maxsub = ax; + if (subleaf == maxsub) break; + } else if (leaf == 11) { + if ((cx & 0xff00) == 0) break; + } else if (leaf == 13) { + if (ax == 0 && bx == 0 && cx == 0 && dx == 0) break; + } else { + // For any other leaf, stop after subleaf 0. + break; + } + } + } + // Iterate over all extended leaves + AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax + uint32_t ext_max = ax; + for (uint32_t leaf = 0x80000001; leaf <= ext_max && leaf <= 0x800000FF; ++leaf) { + AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); + } +} +#endif +} // namespace + +void RandAddDynamicEnv(CSHA512& hasher) +{ + RandAddSeedPerfmon(hasher); + + // Various clocks +#ifdef WIN32 + FILETIME ftime; + GetSystemTimeAsFileTime(&ftime); + hasher << ftime; +#else +# ifndef __MACH__ + // On non-MacOS systems, use various clock_gettime() calls. + struct timespec ts = {}; +# ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &ts); + hasher << ts; +# endif +# ifdef CLOCK_REALTIME + clock_gettime(CLOCK_REALTIME, &ts); + hasher << ts; +# endif +# ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &ts); + hasher << ts; +# endif +# else + // On MacOS use mach_absolute_time (number of CPU ticks since boot) as a replacement for CLOCK_MONOTONIC, + // and clock_get_time for CALENDAR_CLOCK as a replacement for CLOCK_REALTIME. + hasher << mach_absolute_time(); + // From https://gist.github.com/jbenet/1087739 + clock_serv_t cclock; + mach_timespec_t mts = {}; + if (host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock) == KERN_SUCCESS && clock_get_time(cclock, &mts) == KERN_SUCCESS) { + hasher << mts; + mach_port_deallocate(mach_task_self(), cclock); + } +# endif + // gettimeofday is available on all UNIX systems, but only has microsecond precision. + struct timeval tv = {}; + gettimeofday(&tv, nullptr); + hasher << tv; +#endif + // Probably redundant, but also use all the clocks C++11 provides: + hasher << std::chrono::system_clock::now().time_since_epoch().count(); + hasher << std::chrono::steady_clock::now().time_since_epoch().count(); + hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); + +#ifndef WIN32 + // Current resource usage. + struct rusage usage = {}; + if (getrusage(RUSAGE_SELF, &usage) == 0) hasher << usage; +#endif + +#ifdef __linux__ + AddFile(hasher, "/proc/diskstats"); + AddFile(hasher, "/proc/vmstat"); + AddFile(hasher, "/proc/schedstat"); + AddFile(hasher, "/proc/zoneinfo"); + AddFile(hasher, "/proc/meminfo"); + AddFile(hasher, "/proc/softirqs"); + AddFile(hasher, "/proc/stat"); + AddFile(hasher, "/proc/self/schedstat"); + AddFile(hasher, "/proc/self/status"); +#endif + +#if HAVE_SYSCTL +# ifdef CTL_KERN +# if defined(KERN_PROC) && defined(KERN_PROC_ALL) + AddSysctl<CTL_KERN, KERN_PROC, KERN_PROC_ALL>(hasher); +# endif +# endif +# ifdef CTL_HW +# ifdef HW_DISKSTATS + AddSysctl<CTL_HW, HW_DISKSTATS>(hasher); +# endif +# endif +# ifdef CTL_VM +# ifdef VM_LOADAVG + AddSysctl<CTL_VM, VM_LOADAVG>(hasher); +# endif +# ifdef VM_TOTAL + AddSysctl<CTL_VM, VM_TOTAL>(hasher); +# endif +# ifdef VM_METER + AddSysctl<CTL_VM, VM_METER>(hasher); +# endif +# endif +#endif + + // Stack and heap location + void* addr = malloc(4097); + hasher << &addr << addr; + free(addr); +} + +void RandAddStaticEnv(CSHA512& hasher) +{ + // Some compile-time static properties + hasher << (CHAR_MIN < 0) << sizeof(void*) << sizeof(long) << sizeof(int); +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + hasher << __GNUC__ << __GNUC_MINOR__ << __GNUC_PATCHLEVEL__; +#endif +#ifdef _MSC_VER + hasher << _MSC_VER; +#endif + hasher << __cplusplus; +#ifdef _XOPEN_VERSION + hasher << _XOPEN_VERSION; +#endif +#ifdef __VERSION__ + const char* COMPILER_VERSION = __VERSION__; + hasher.Write((const unsigned char*)COMPILER_VERSION, strlen(COMPILER_VERSION) + 1); +#endif + + // Bitcoin client version + hasher << CLIENT_VERSION; + +#ifdef __linux__ + // Information available through getauxval() +# ifdef AT_HWCAP + hasher << getauxval(AT_HWCAP); +# endif +# ifdef AT_HWCAP2 + hasher << getauxval(AT_HWCAP2); +# endif +# ifdef AT_RANDOM + const unsigned char* random_aux = (const unsigned char*)getauxval(AT_RANDOM); + if (random_aux) hasher.Write(random_aux, 16); +# endif +# ifdef AT_PLATFORM + const char* platform_str = (const char*)getauxval(AT_PLATFORM); + if (platform_str) hasher.Write((const unsigned char*)platform_str, strlen(platform_str) + 1); +# endif +# ifdef AT_EXECFN + const char* exec_str = (const char*)getauxval(AT_EXECFN); + if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); +# endif +#endif // __linux__ + +#ifdef HAVE_GETCPUID + AddAllCPUID(hasher); +#endif + + // Memory locations + hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; + + // Hostname + char hname[256]; + if (gethostname(hname, 256) == 0) { + hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); + } + +#if HAVE_DECL_GETIFADDRS + // Network interfaces + struct ifaddrs *ifad = NULL; + getifaddrs(&ifad); + struct ifaddrs *ifit = ifad; + while (ifit != NULL) { + hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); + hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); + hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); + AddSockaddr(hasher, ifit->ifa_addr); + AddSockaddr(hasher, ifit->ifa_netmask); + AddSockaddr(hasher, ifit->ifa_dstaddr); + ifit = ifit->ifa_next; + } + freeifaddrs(ifad); +#endif + +#ifndef WIN32 + // UNIX kernel information + struct utsname name; + if (uname(&name) != -1) { + hasher.Write((const unsigned char*)&name.sysname, strlen(name.sysname) + 1); + hasher.Write((const unsigned char*)&name.nodename, strlen(name.nodename) + 1); + hasher.Write((const unsigned char*)&name.release, strlen(name.release) + 1); + hasher.Write((const unsigned char*)&name.version, strlen(name.version) + 1); + hasher.Write((const unsigned char*)&name.machine, strlen(name.machine) + 1); + } + + /* Path and filesystem provided data */ + AddPath(hasher, "/"); + AddPath(hasher, "."); + AddPath(hasher, "/tmp"); + AddPath(hasher, "/home"); + AddPath(hasher, "/proc"); +#ifdef __linux__ + AddFile(hasher, "/proc/cmdline"); + AddFile(hasher, "/proc/cpuinfo"); + AddFile(hasher, "/proc/version"); +#endif + AddFile(hasher, "/etc/passwd"); + AddFile(hasher, "/etc/group"); + AddFile(hasher, "/etc/hosts"); + AddFile(hasher, "/etc/resolv.conf"); + AddFile(hasher, "/etc/timezone"); + AddFile(hasher, "/etc/localtime"); +#endif + + // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these + // will exist on every system. +#if HAVE_SYSCTL +# ifdef CTL_HW +# ifdef HW_MACHINE + AddSysctl<CTL_HW, HW_MACHINE>(hasher); +# endif +# ifdef HW_MODEL + AddSysctl<CTL_HW, HW_MODEL>(hasher); +# endif +# ifdef HW_NCPU + AddSysctl<CTL_HW, HW_NCPU>(hasher); +# endif +# ifdef HW_PHYSMEM + AddSysctl<CTL_HW, HW_PHYSMEM>(hasher); +# endif +# ifdef HW_USERMEM + AddSysctl<CTL_HW, HW_USERMEM>(hasher); +# endif +# ifdef HW_MACHINE_ARCH + AddSysctl<CTL_HW, HW_MACHINE_ARCH>(hasher); +# endif +# ifdef HW_REALMEM + AddSysctl<CTL_HW, HW_REALMEM>(hasher); +# endif +# ifdef HW_CPU_FREQ + AddSysctl<CTL_HW, HW_CPU_FREQ>(hasher); +# endif +# ifdef HW_BUS_FREQ + AddSysctl<CTL_HW, HW_BUS_FREQ>(hasher); +# endif +# ifdef HW_CACHELINE + AddSysctl<CTL_HW, HW_CACHELINE>(hasher); +# endif +# endif +# ifdef CTL_KERN +# ifdef KERN_BOOTFILE + AddSysctl<CTL_KERN, KERN_BOOTFILE>(hasher); +# endif +# ifdef KERN_BOOTTIME + AddSysctl<CTL_KERN, KERN_BOOTTIME>(hasher); +# endif +# ifdef KERN_CLOCKRATE + AddSysctl<CTL_KERN, KERN_CLOCKRATE>(hasher); +# endif +# ifdef KERN_HOSTID + AddSysctl<CTL_KERN, KERN_HOSTID>(hasher); +# endif +# ifdef KERN_HOSTUUID + AddSysctl<CTL_KERN, KERN_HOSTUUID>(hasher); +# endif +# ifdef KERN_HOSTNAME + AddSysctl<CTL_KERN, KERN_HOSTNAME>(hasher); +# endif +# ifdef KERN_OSRELDATE + AddSysctl<CTL_KERN, KERN_OSRELDATE>(hasher); +# endif +# ifdef KERN_OSRELEASE + AddSysctl<CTL_KERN, KERN_OSRELEASE>(hasher); +# endif +# ifdef KERN_OSREV + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_OSTYPE + AddSysctl<CTL_KERN, KERN_OSTYPE>(hasher); +# endif +# ifdef KERN_POSIX1 + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_VERSION + AddSysctl<CTL_KERN, KERN_VERSION>(hasher); +# endif +# endif +#endif + + // Env variables + if (environ) { + for (size_t i = 0; environ[i]; ++i) { + hasher.Write((const unsigned char*)environ[i], strlen(environ[i])); + } + } + + // Process, thread, user, session, group, ... ids. +#ifdef WIN32 + hasher << GetCurrentProcessId() << GetCurrentThreadId(); +#else + hasher << getpid() << getppid() << getsid(0) << getpgid(0) << getuid() << geteuid() << getgid() << getegid(); +#endif + hasher << std::this_thread::get_id(); +} diff --git a/src/randomenv.h b/src/randomenv.h new file mode 100644 index 0000000000..46cea6f6f2 --- /dev/null +++ b/src/randomenv.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOMENV_H +#define BITCOIN_RANDOMENV_H + +#include <crypto/sha512.h> + +/** Gather non-cryptographic environment data that changes over time. */ +void RandAddDynamicEnv(CSHA512& hasher); + +/** Gather non-cryptographic environment data that does not change over time. */ +void RandAddStaticEnv(CSHA512& hasher); + +#endif diff --git a/src/rest.cpp b/src/rest.cpp index 228c122de3..0629557584 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <core_io.h> #include <httpserver.h> #include <index/txindex.h> +#include <node/context.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <rpc/blockchain.h> @@ -16,6 +17,7 @@ #include <streams.h> #include <sync.h> #include <txmempool.h> +#include <util/check.h> #include <util/strencodings.h> #include <validation.h> #include <version.h> @@ -69,6 +71,24 @@ static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string me return false; } +/** + * Get the node context mempool. + * + * Set the HTTP error and return nullptr if node context + * mempool is not found. + * + * @param[in] req the HTTP request + * return pointer to the mempool or nullptr if no mempool found + */ +static CTxMemPool* GetMemPool(HTTPRequest* req) +{ + if (!g_rpc_node || !g_rpc_node->mempool) { + RESTERR(req, HTTP_NOT_FOUND, "Mempool disabled or instance not found"); + return nullptr; + } + return g_rpc_node->mempool; +} + static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) { const std::string::size_type pos = strReq.rfind('.'); @@ -295,12 +315,14 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) { if (!CheckWarmup(req)) return false; + const CTxMemPool* mempool = GetMemPool(req); + if (!mempool) return false; std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RetFormat::JSON: { - UniValue mempoolInfoObject = MempoolInfoToJSON(::mempool); + UniValue mempoolInfoObject = MempoolInfoToJSON(*mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -315,14 +337,15 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPart) { - if (!CheckWarmup(req)) - return false; + if (!CheckWarmup(req)) return false; + const CTxMemPool* mempool = GetMemPool(req); + if (!mempool) return false; std::string param; const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RetFormat::JSON: { - UniValue mempoolObject = MempoolToJSON(::mempool, true); + UniValue mempoolObject = MempoolToJSON(*mempool, true); std::string strJSON = mempoolObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -500,11 +523,13 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) }; if (fCheckMemPool) { + const CTxMemPool* mempool = GetMemPool(req); + if (!mempool) return false; // use db+mempool as cache backend in case user likes to query mempool - LOCK2(cs_main, mempool.cs); + LOCK2(cs_main, mempool->cs); CCoinsViewCache& viewChain = ::ChainstateActive().CoinsTip(); - CCoinsViewMemPool viewMempool(&viewChain, mempool); - process_utxos(viewMempool, mempool); + CCoinsViewMemPool viewMempool(&viewChain, *mempool); + process_utxos(viewMempool, *mempool); } else { LOCK(cs_main); // no need to lock mempool! process_utxos(::ChainstateActive().CoinsTip(), CTxMemPool()); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4bbd4aaf64..eb5148eebd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -10,11 +10,13 @@ #include <chain.h> #include <chainparams.h> #include <coins.h> -#include <node/coinstats.h> #include <consensus/validation.h> #include <core_io.h> #include <hash.h> #include <index/blockfilterindex.h> +#include <node/coinstats.h> +#include <node/context.h> +#include <node/utxo_snapshot.h> #include <policy/feerate.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -34,13 +36,10 @@ #include <validationinterface.h> #include <warnings.h> -#include <assert.h> #include <stdint.h> #include <univalue.h> -#include <boost/thread/thread.hpp> // boost::thread::interrupt - #include <condition_variable> #include <memory> #include <mutex> @@ -55,11 +54,20 @@ static Mutex cs_blockchange; static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock; +CTxMemPool& EnsureMemPool() +{ + CHECK_NONFATAL(g_rpc_node); + if (!g_rpc_node->mempool) { + throw JSONRPCError(RPC_CLIENT_MEMPOOL_DISABLED, "Mempool disabled or instance not found"); + } + return *g_rpc_node->mempool; +} + /* Calculate the difficulty for a given block index. */ double GetDifficulty(const CBlockIndex* blockindex) { - assert(blockindex); + CHECK_NONFATAL(blockindex); int nShift = (blockindex->nBits >> 24) & 0xff; double dDiff = @@ -520,7 +528,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) if (!request.params[0].isNull()) fVerbose = request.params[0].get_bool(); - return MempoolToJSON(::mempool, fVerbose); + return MempoolToJSON(EnsureMemPool(), fVerbose); } static UniValue getmempoolancestors(const JSONRPCRequest& request) @@ -558,6 +566,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); + const CTxMemPool& mempool = EnsureMemPool(); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -583,7 +592,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *ancestorIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(::mempool, info, e); + entryToJSON(mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -625,6 +634,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); + const CTxMemPool& mempool = EnsureMemPool(); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -650,7 +660,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *descendantIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(::mempool, info, e); + entryToJSON(mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -677,6 +687,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) uint256 hash = ParseHashV(request.params[0], "parameter 1"); + const CTxMemPool& mempool = EnsureMemPool(); LOCK(mempool.cs); CTxMemPool::txiter it = mempool.mapTx.find(hash); @@ -686,7 +697,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(::mempool, info, e); + entryToJSON(mempool, info, e); return info; } @@ -734,8 +745,8 @@ static UniValue getblockheader(const JSONRPCRequest& request) " \"version\" : n, (numeric) The block version\n" " \"versionHex\" : \"00000000\", (string) The block version formatted in hexadecimal\n" " \"merkleroot\" : \"xxxx\", (string) The merkle root\n" - " \"time\" : ttt, (numeric) The block time in seconds since epoch (Jan 1 1970 GMT)\n" - " \"mediantime\" : ttt, (numeric) The median block time in seconds since epoch (Jan 1 1970 GMT)\n" + " \"time\" : ttt, (numeric) The block time expressed in " + UNIX_EPOCH_TIME + "\n" + " \"mediantime\" : ttt, (numeric) The median block time expressed in " + UNIX_EPOCH_TIME + "\n" " \"nonce\" : n, (numeric) The nonce\n" " \"bits\" : \"1d00ffff\", (string) The bits\n" " \"difficulty\" : x.xxx, (numeric) The difficulty\n" @@ -846,8 +857,8 @@ static UniValue getblock(const JSONRPCRequest& request) " \"transactionid\" (string) The transaction id\n" " ,...\n" " ],\n" - " \"time\" : ttt, (numeric) The block time in seconds since epoch (Jan 1 1970 GMT)\n" - " \"mediantime\" : ttt, (numeric) The median block time in seconds since epoch (Jan 1 1970 GMT)\n" + " \"time\" : ttt, (numeric) The block time expressed in " + UNIX_EPOCH_TIME + "\n" + " \"mediantime\" : ttt, (numeric) The median block time expressed in " + UNIX_EPOCH_TIME + "\n" " \"nonce\" : n, (numeric) The nonce\n" " \"bits\" : \"1d00ffff\", (string) The bits\n" " \"difficulty\" : x.xxx, (numeric) The difficulty\n" @@ -913,7 +924,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) { RPCHelpMan{"pruneblockchain", "", { - {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The block height to prune up to. May be set to a discrete height, or a unix timestamp\n" + {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The block height to prune up to. May be set to a discrete height, or to a " + UNIX_EPOCH_TIME + "\n" " to prune blocks whose block time is at least 2 hours older than the provided timestamp."}, }, RPCResult{ @@ -958,7 +969,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) PruneBlockFilesManual(height); const CBlockIndex* block = ::ChainActive().Tip(); - assert(block); + CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { block = block->pprev; } @@ -1062,6 +1073,7 @@ UniValue gettxout(const JSONRPCRequest& request) CCoinsViewCache* coins_view = &::ChainstateActive().CoinsTip(); if (fMempool) { + const CTxMemPool& mempool = EnsureMemPool(); LOCK(mempool.cs); CCoinsViewMemPool view(coins_view, mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { @@ -1192,7 +1204,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) {}, RPCResult{ "{\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"chain\": \"xxxx\", (string) current network name (main, test, regtest)\n" " \"blocks\": xxxxxx, (numeric) the height of the most-work fully-validated chain. The genesis block has height 0\n" " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" @@ -1253,7 +1265,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.pushKV("pruned", fPruneMode); if (fPruneMode) { const CBlockIndex* block = tip; - assert(block); + CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { block = block->pprev; } @@ -1278,7 +1290,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) BIP9SoftForkDescPushBack(softforks, "testdummy", consensusParams, Consensus::DEPLOYMENT_TESTDUMMY); obj.pushKV("softforks", softforks); - obj.pushKV("warnings", GetWarnings("statusbar")); + obj.pushKV("warnings", GetWarnings(false)); return obj; } @@ -1440,7 +1452,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) }, }.Check(request); - return MempoolInfoToJSON(::mempool); + return MempoolInfoToJSON(EnsureMemPool()); } static UniValue preciousblock(const JSONRPCRequest& request) @@ -1470,7 +1482,7 @@ static UniValue preciousblock(const JSONRPCRequest& request) } } - CValidationState state; + BlockValidationState state; PreciousBlock(state, Params(), pblockindex); if (!state.IsValid()) { @@ -1495,7 +1507,7 @@ static UniValue invalidateblock(const JSONRPCRequest& request) }.Check(request); uint256 hash(ParseHashV(request.params[0], "blockhash")); - CValidationState state; + BlockValidationState state; CBlockIndex* pblockindex; { @@ -1545,7 +1557,7 @@ static UniValue reconsiderblock(const JSONRPCRequest& request) ResetBlockFailureFlags(pblockindex); } - CValidationState state; + BlockValidationState state; ActivateBestChain(state, Params()); if (!state.IsValid()) { @@ -1565,7 +1577,7 @@ static UniValue getchaintxstats(const JSONRPCRequest& request) }, RPCResult{ "{\n" - " \"time\": xxxxx, (numeric) The timestamp for the final block in the window in UNIX format.\n" + " \"time\": xxxxx, (numeric) The timestamp for the final block in the window, expressed in " + UNIX_EPOCH_TIME + ".\n" " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" " \"window_final_block_height\": xxxxx, (numeric) The height of the final block in the window.\n" @@ -1599,7 +1611,7 @@ static UniValue getchaintxstats(const JSONRPCRequest& request) } } - assert(pindex != nullptr); + CHECK_NONFATAL(pindex != nullptr); if (request.params[0].isNull()) { blockcount = std::max(0, std::min(blockcount, pindex->nHeight - 1)); @@ -1772,7 +1784,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) } } - assert(pindex != nullptr); + CHECK_NONFATAL(pindex != nullptr); std::set<std::string> stats; if (!request.params[1].isNull()) { @@ -1872,7 +1884,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) } CAmount txfee = tx_total_in - tx_total_out; - assert(MoneyRange(txfee)); + CHECK_NONFATAL(MoneyRange(txfee)); if (do_medianfee) { fee_array.push_back(txfee); } @@ -1956,11 +1968,13 @@ static UniValue savemempool(const JSONRPCRequest& request) }, }.Check(request); - if (!::mempool.IsLoaded()) { + const CTxMemPool& mempool = EnsureMemPool(); + + if (!mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool(::mempool)) { + if (!DumpMempool(mempool)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } @@ -1976,7 +1990,6 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - boost::this_thread::interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -2009,7 +2022,7 @@ public: explicit CoinsViewScanReserver() : m_could_reserve(false) {} bool reserve() { - assert (!m_could_reserve); + CHECK_NONFATAL(!m_could_reserve); std::lock_guard<std::mutex> lock(g_utxosetscan); if (g_scan_in_progress) { return false; @@ -2048,7 +2061,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) " \"start\" for starting a scan\n" " \"abort\" for aborting the current scan (returns true when abort was successful)\n" " \"status\" for progress report (in %) of the current scan"}, - {"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::NO, "Array of scan objects\n" + {"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" " Every scan object is either a string descriptor or an object:", { {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, @@ -2108,6 +2121,11 @@ UniValue scantxoutset(const JSONRPCRequest& request) if (!reserver.reserve()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); } + + if (request.params.size() < 2) { + throw JSONRPCError(RPC_MISC_ERROR, "scanobjects argument is required for the start action"); + } + std::set<CScript> needles; std::map<CScript, std::string> descriptors; CAmount total_in = 0; @@ -2136,9 +2154,9 @@ UniValue scantxoutset(const JSONRPCRequest& request) LOCK(cs_main); ::ChainstateActive().ForceFlushStateToDisk(); pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); - assert(pcursor); + CHECK_NONFATAL(pcursor); tip = ::ChainActive().Tip(); - assert(tip); + CHECK_NONFATAL(tip); } bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); result.pushKV("success", res); @@ -2246,6 +2264,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + "{\n" + " \"coins_written\": n, (numeric) the number of coins written in the snapshot\n" + " \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n" + " \"base_height\": n, (string) the height of the base of the snapshot\n" + " \"path\": \"...\" (string) the absolute path that the snapshot was written to\n" + "]\n" + }, + RPCExamples{ + HelpExampleCli("dumptxoutset", "utxo.dat") + } + }.Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE* file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr<CCoinsViewCursor> pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats + // based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2282,6 +2407,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on @@ -2290,3 +2416,5 @@ void RegisterBlockchainRPCCommands(CRPCTable &t) for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } + +NodeContext* g_rpc_node = nullptr; diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index ff461fbcbc..a02e5fae0e 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,6 +17,7 @@ class CBlock; class CBlockIndex; class CTxMemPool; class UniValue; +struct NodeContext; static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -46,4 +47,11 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); +//! Pointer to node state that needs to be declared as a global to be accessible +//! RPC methods. Due to limitations of the RPC framework, there's currently no +//! direct way to pass in state to RPC methods without globals. +extern NodeContext* g_rpc_node; + +CTxMemPool& EnsureMemPool(); + #endif diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 32e18312e1..2eaa3427eb 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -30,6 +30,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "utxoupdatepsbt", 1, "descriptors" }, { "generatetoaddress", 0, "nblocks" }, { "generatetoaddress", 2, "maxtries" }, + { "generatetodescriptor", 0, "num_blocks" }, + { "generatetodescriptor", 2, "maxtries" }, { "getnetworkhashps", 0, "nblocks" }, { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 07c2958635..e5994b172b 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,12 +13,15 @@ #include <key_io.h> #include <miner.h> #include <net.h> +#include <node/context.h> #include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/descriptor.h> #include <script/script.h> +#include <script/signingprovider.h> #include <shutdown.h> #include <txmempool.h> #include <univalue.h> @@ -99,7 +102,7 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } -static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) +static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; int nHeight = 0; @@ -113,7 +116,7 @@ static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, ui UniValue blockHashes(UniValue::VARR); while (nHeight < nHeightEnd && !ShutdownRequested()) { - std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(Params()).CreateNewBlock(coinbase_script)); + std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(mempool, Params()).CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; @@ -140,6 +143,49 @@ static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, ui return blockHashes; } +static UniValue generatetodescriptor(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "generatetodescriptor", + "\nMine blocks immediately to a specified descriptor (before the RPC call returns)\n", + { + {"num_blocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor to send the newly generated bitcoin to."}, + {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, + }, + RPCResult{ + "[ blockhashes ] (array) hashes of blocks generated\n"}, + RPCExamples{ + "\nGenerate 11 blocks to mydesc\n" + HelpExampleCli("generatetodescriptor", "11 \"mydesc\"")}, + } + .Check(request); + + const int num_blocks{request.params[0].get_int()}; + const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; + + FlatSigningProvider key_provider; + std::string error; + const auto desc = Parse(request.params[1].get_str(), key_provider, error, /* require_checksum = */ false); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + if (desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); + } + + FlatSigningProvider provider; + std::vector<CScript> coinbase_script; + if (!desc->Expand(0, key_provider, coinbase_script, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + const CTxMemPool& mempool = EnsureMemPool(); + + CHECK_NONFATAL(coinbase_script.size() == 1); + + return generateBlocks(mempool, coinbase_script.at(0), num_blocks, max_tries); +} + static UniValue generatetoaddress(const JSONRPCRequest& request) { RPCHelpMan{"generatetoaddress", @@ -171,9 +217,11 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); } + const CTxMemPool& mempool = EnsureMemPool(); + CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(coinbase_script, nGenerate, nMaxTries); + return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); } static UniValue getmininginfo(const JSONRPCRequest& request) @@ -189,7 +237,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"networkhashps\": nnn, (numeric) The network hashes per second\n" " \"pooledtx\": n (numeric) The size of the mempool\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"chain\": \"xxxx\", (string) current network name (main, test, regtest)\n" " \"warnings\": \"...\" (string) any network and blockchain warnings\n" "}\n" }, @@ -200,6 +248,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) }.Check(request); LOCK(cs_main); + const CTxMemPool& mempool = EnsureMemPool(); UniValue obj(UniValue::VOBJ); obj.pushKV("blocks", (int)::ChainActive().Height()); @@ -209,7 +258,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) obj.pushKV("networkhashps", getnetworkhashps(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); obj.pushKV("chain", Params().NetworkIDString()); - obj.pushKV("warnings", GetWarnings("statusbar")); + obj.pushKV("warnings", GetWarnings(false)); return obj; } @@ -246,13 +295,13 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0."); } - mempool.PrioritiseTransaction(hash, nAmount); + EnsureMemPool().PrioritiseTransaction(hash, nAmount); return true; } // NOTE: Assumes a conclusive result; if result is inconclusive, it must be handled by caller -static UniValue BIP22ValidationResult(const CValidationState& state) +static UniValue BIP22ValidationResult(const BlockValidationState& state) { if (state.IsValid()) return NullUniValue; @@ -331,13 +380,11 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) " }\n" " ,...\n" " ],\n" - " \"coinbaseaux\" : { (json object) data that should be included in the coinbase's scriptSig content\n" - " \"flags\" : \"xx\" (string) key name is to be ignored, and value included in scriptSig\n" - " },\n" + " \"coinbaseaux\" : { ... }, (json object) data that should be included in the coinbase's scriptSig content\n" " \"coinbasevalue\" : n, (numeric) maximum allowable input to coinbase transaction, including the generation award and transaction fees (in satoshis)\n" " \"coinbasetxn\" : { ... }, (json object) information for coinbase transaction\n" " \"target\" : \"xxxx\", (string) The hash target\n" - " \"mintime\" : xxx, (numeric) The minimum timestamp appropriate for next block time in seconds since epoch (Jan 1 1970 GMT)\n" + " \"mintime\" : xxx, (numeric) The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME + "\n" " \"mutable\" : [ (array of string) list of ways the block template may be changed \n" " \"value\" (string) A way the block template may be changed, e.g. 'time', 'transactions', 'prevblock'\n" " ,...\n" @@ -346,7 +393,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) " \"sigoplimit\" : n, (numeric) limit of sigops in blocks\n" " \"sizelimit\" : n, (numeric) limit of block size\n" " \"weightlimit\" : n, (numeric) limit of block weight\n" - " \"curtime\" : ttt, (numeric) current timestamp in seconds since epoch (Jan 1 1970 GMT)\n" + " \"curtime\" : ttt, (numeric) current timestamp in " + UNIX_EPOCH_TIME + "\n" " \"bits\" : \"xxxxxxxx\", (string) compressed target of next block\n" " \"height\" : n (numeric) The height of the next block\n" "}\n" @@ -401,7 +448,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) // TestBlockValidity only supports blocks built on the current Tip if (block.hashPrevBlock != pindexPrev->GetBlockHash()) return "inconclusive-not-best-prevblk"; - CValidationState state; + BlockValidationState state; TestBlockValidity(state, Params(), block, pindexPrev, false, true); return BIP22ValidationResult(state); } @@ -424,16 +471,17 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) if (strMode != "template") throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - if (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) + if (g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (::ChainstateActive().IsInitialBlockDownload()) throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); static unsigned int nTransactionsUpdatedLast; + const CTxMemPool& mempool = EnsureMemPool(); if (!lpval.isNull()) { @@ -468,7 +516,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) if (g_best_block_cv.wait_until(lock, checktxtime) == std::cv_status::timeout) { // Timeout: Check transactions for update - // without holding ::mempool.cs to avoid deadlocks + // without holding the mempool lock to avoid deadlocks if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP) break; checktxtime += std::chrono::seconds(10); @@ -504,14 +552,14 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) // Create new block CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = BlockAssembler(Params()).CreateNewBlock(scriptDummy); + pblocktemplate = BlockAssembler(mempool, Params()).CreateNewBlock(scriptDummy); if (!pblocktemplate) throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); // Need to update only after we know CreateNewBlock succeeded pindexPrev = pindexPrevNew; } - assert(pindexPrev); + CHECK_NONFATAL(pindexPrev); CBlock* pblock = &pblocktemplate->block; // pointer for convenience const Consensus::Params& consensusParams = Params().GetConsensus(); @@ -553,7 +601,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) entry.pushKV("fee", pblocktemplate->vTxFees[index_in_template]); int64_t nTxSigOps = pblocktemplate->vTxSigOpsCost[index_in_template]; if (fPreSegWit) { - assert(nTxSigOps % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nTxSigOps % WITNESS_SCALE_FACTOR == 0); nTxSigOps /= WITNESS_SCALE_FACTOR; } entry.pushKV("sigops", nTxSigOps); @@ -563,7 +611,6 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) } UniValue aux(UniValue::VOBJ); - aux.pushKV("flags", HexStr(COINBASE_FLAGS.begin(), COINBASE_FLAGS.end())); arith_uint256 hashTarget = arith_uint256().SetCompact(pblock->nBits); @@ -642,9 +689,9 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST; int64_t nSizeLimit = MAX_BLOCK_SERIALIZED_SIZE; if (fPreSegWit) { - assert(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); nSigOpLimit /= WITNESS_SCALE_FACTOR; - assert(nSizeLimit % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nSizeLimit % WITNESS_SCALE_FACTOR == 0); nSizeLimit /= WITNESS_SCALE_FACTOR; } result.pushKV("sigoplimit", nSigOpLimit); @@ -668,12 +715,12 @@ class submitblock_StateCatcher : public CValidationInterface public: uint256 hash; bool found; - CValidationState state; + BlockValidationState state; explicit submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {} protected: - void BlockChecked(const CBlock& block, const CValidationState& stateIn) override { + void BlockChecked(const CBlock& block, const BlockValidationState& stateIn) override { if (block.GetHash() != hash) return; found = true; @@ -772,8 +819,8 @@ static UniValue submitheader(const JSONRPCRequest& request) } } - CValidationState state; - ProcessNewBlockHeaders({h}, state, Params(), /* ppindex */ nullptr, /* first_invalid */ nullptr); + BlockValidationState state; + ProcessNewBlockHeaders({h}, state, Params()); if (state.IsValid()) return NullUniValue; if (state.IsError()) { throw JSONRPCError(RPC_VERIFY_ERROR, FormatStateMessage(state)); @@ -961,6 +1008,7 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, + { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index d289274a37..56bd33b0ec 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -1,17 +1,18 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <key_io.h> #include <httpserver.h> +#include <key_io.h> #include <outputtype.h> #include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/util.h> #include <script/descriptor.h> -#include <util/system.h> +#include <util/check.h> #include <util/strencodings.h> +#include <util/system.h> #include <util/validation.h> #include <stdint.h> @@ -342,7 +343,7 @@ static UniValue setmocktime(const JSONRPCRequest& request) RPCHelpMan{"setmocktime", "\nSet the local time to given timestamp (-regtest only)\n", { - {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Unix seconds-since-epoch timestamp\n" + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" " Pass 0 to go back to using the system time."}, }, RPCResults{}, @@ -540,6 +541,7 @@ static UniValue echo(const JSONRPCRequest& request) throw std::runtime_error( RPCHelpMan{"echo|echojson ...", "\nSimply echo back the input arguments. This command is for testing.\n" + "\nIt will return an internal bug report when exactly 100 arguments are passed.\n" "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in " "bitcoin-cli and the GUI. There is no server-side difference.", {}, @@ -548,6 +550,8 @@ static UniValue echo(const JSONRPCRequest& request) }.ToString() ); + CHECK_NONFATAL(request.params.size() != 100); + return request.params; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 7b1507e4dc..1ce49709b2 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -8,10 +8,13 @@ #include <clientversion.h> #include <core_io.h> #include <net.h> -#include <net_processing.h> #include <net_permissions.h> +#include <net_processing.h> +#include <net_types.h> // For banmap_t #include <netbase.h> +#include <node/context.h> #include <policy/settings.h> +#include <rpc/blockchain.h> #include <rpc/protocol.h> #include <rpc/util.h> #include <sync.h> @@ -38,10 +41,10 @@ static UniValue getconnectioncount(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - return (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL); + return (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL); } static UniValue ping(const JSONRPCRequest& request) @@ -58,11 +61,11 @@ static UniValue ping(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); // Request that each node send a ping during next message processing pass - g_connman->ForEachNode([](CNode* pnode) { + g_rpc_node->connman->ForEachNode([](CNode* pnode) { pnode->fPingQueued = true; }); return NullUniValue; @@ -80,17 +83,18 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) " \"addr\":\"host:port\", (string) The IP address and port of the peer\n" " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" + " \"mapped_as\":\"mapped_as\", (string) The AS in the BGP route to the peer used for diversifying peer selection\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"servicesnames\":[ (array) the services offered, in human-readable form\n" " \"SERVICE_NAME\", (string) the service name if it is recognised\n" " ...\n" " ],\n" " \"relaytxes\":true|false, (boolean) Whether peer has asked us to relay transactions to it\n" - " \"lastsend\": ttt, (numeric) The time in seconds since epoch (Jan 1 1970 GMT) of the last send\n" - " \"lastrecv\": ttt, (numeric) The time in seconds since epoch (Jan 1 1970 GMT) of the last receive\n" + " \"lastsend\": ttt, (numeric) The " + UNIX_EPOCH_TIME + " of the last send\n" + " \"lastrecv\": ttt, (numeric) The " + UNIX_EPOCH_TIME + " of the last receive\n" " \"bytessent\": n, (numeric) The total bytes sent\n" " \"bytesrecv\": n, (numeric) The total bytes received\n" - " \"conntime\": ttt, (numeric) The connection time in seconds since epoch (Jan 1 1970 GMT)\n" + " \"conntime\": ttt, (numeric) The " + UNIX_EPOCH_TIME + " of the connection\n" " \"timeoffset\": ttt, (numeric) The time offset in seconds\n" " \"pingtime\": n, (numeric) ping time (if available)\n" " \"minping\": n, (numeric) minimum observed ping time (if any at all)\n" @@ -131,11 +135,11 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::vector<CNodeStats> vstats; - g_connman->GetNodeStats(vstats); + g_rpc_node->connman->GetNodeStats(vstats); UniValue ret(UniValue::VARR); @@ -149,6 +153,9 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("addrlocal", stats.addrLocal); if (stats.addrBind.IsValid()) obj.pushKV("addrbind", stats.addrBind.ToString()); + if (stats.m_mapped_as != 0) { + obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); + } obj.pushKV("services", strprintf("%016x", stats.nServices)); obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); obj.pushKV("relaytxes", stats.fRelayTxes); @@ -234,7 +241,7 @@ static UniValue addnode(const JSONRPCRequest& request) }, }.ToString()); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::string strNode = request.params[0].get_str(); @@ -242,18 +249,18 @@ static UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); + g_rpc_node->connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } if (strCommand == "add") { - if(!g_connman->AddNode(strNode)) + if(!g_rpc_node->connman->AddNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Node already added"); } else if(strCommand == "remove") { - if(!g_connman->RemoveAddedNode(strNode)) + if(!g_rpc_node->connman->RemoveAddedNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); } @@ -279,7 +286,7 @@ static UniValue disconnectnode(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); bool success; @@ -288,11 +295,11 @@ static UniValue disconnectnode(const JSONRPCRequest& request) if (!address_arg.isNull() && id_arg.isNull()) { /* handle disconnect-by-address */ - success = g_connman->DisconnectNode(address_arg.get_str()); + success = g_rpc_node->connman->DisconnectNode(address_arg.get_str()); } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { /* handle disconnect-by-id */ NodeId nodeid = (NodeId) id_arg.get_int64(); - success = g_connman->DisconnectNode(nodeid); + success = g_rpc_node->connman->DisconnectNode(nodeid); } else { throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); } @@ -333,10 +340,10 @@ static UniValue getaddednodeinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - std::vector<AddedNodeInfo> vInfo = g_connman->GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = g_rpc_node->connman->GetAddedNodeInfo(); if (!request.params[0].isNull()) { bool found = false; @@ -399,21 +406,21 @@ static UniValue getnettotals(const JSONRPCRequest& request) + HelpExampleRpc("getnettotals", "") }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); UniValue obj(UniValue::VOBJ); - obj.pushKV("totalbytesrecv", g_connman->GetTotalBytesRecv()); - obj.pushKV("totalbytessent", g_connman->GetTotalBytesSent()); + obj.pushKV("totalbytesrecv", g_rpc_node->connman->GetTotalBytesRecv()); + obj.pushKV("totalbytessent", g_rpc_node->connman->GetTotalBytesSent()); obj.pushKV("timemillis", GetTimeMillis()); UniValue outboundLimit(UniValue::VOBJ); - outboundLimit.pushKV("timeframe", g_connman->GetMaxOutboundTimeframe()); - outboundLimit.pushKV("target", g_connman->GetMaxOutboundTarget()); - outboundLimit.pushKV("target_reached", g_connman->OutboundTargetReached(false)); - outboundLimit.pushKV("serve_historical_blocks", !g_connman->OutboundTargetReached(true)); - outboundLimit.pushKV("bytes_left_in_cycle", g_connman->GetOutboundTargetBytesLeft()); - outboundLimit.pushKV("time_left_in_cycle", g_connman->GetMaxOutboundTimeLeftInCycle()); + outboundLimit.pushKV("timeframe", g_rpc_node->connman->GetMaxOutboundTimeframe()); + outboundLimit.pushKV("target", g_rpc_node->connman->GetMaxOutboundTarget()); + outboundLimit.pushKV("target_reached", g_rpc_node->connman->OutboundTargetReached(false)); + outboundLimit.pushKV("serve_historical_blocks", !g_rpc_node->connman->OutboundTargetReached(true)); + outboundLimit.pushKV("bytes_left_in_cycle", g_rpc_node->connman->GetOutboundTargetBytesLeft()); + outboundLimit.pushKV("time_left_in_cycle", g_rpc_node->connman->GetMaxOutboundTimeLeftInCycle()); obj.pushKV("uploadtarget", outboundLimit); return obj; } @@ -492,16 +499,16 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) obj.pushKV("version", CLIENT_VERSION); obj.pushKV("subversion", strSubVersion); obj.pushKV("protocolversion",PROTOCOL_VERSION); - if (g_connman) { - ServiceFlags services = g_connman->GetLocalServices(); + if (g_rpc_node->connman) { + ServiceFlags services = g_rpc_node->connman->GetLocalServices(); obj.pushKV("localservices", strprintf("%016x", services)); obj.pushKV("localservicesnames", GetServicesNames(services)); } obj.pushKV("localrelay", g_relay_txes); obj.pushKV("timeoffset", GetTimeOffset()); - if (g_connman) { - obj.pushKV("networkactive", g_connman->GetNetworkActive()); - obj.pushKV("connections", (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); + if (g_rpc_node->connman) { + obj.pushKV("networkactive", g_rpc_node->connman->GetNetworkActive()); + obj.pushKV("connections", (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); } obj.pushKV("networks", GetNetworksInfo()); obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -519,7 +526,7 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) } } obj.pushKV("localaddresses", localAddresses); - obj.pushKV("warnings", GetWarnings("statusbar")); + obj.pushKV("warnings", GetWarnings(false)); return obj; } @@ -531,7 +538,7 @@ static UniValue setban(const JSONRPCRequest& request) {"subnet", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP/Subnet (see getpeerinfo for nodes IP) with an optional netmask (default is /32 = single IP)"}, {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "'add' to add an IP/Subnet to the list, 'remove' to remove an IP/Subnet from the list"}, {"bantime", RPCArg::Type::NUM, /* default */ "0", "time in seconds how long (or until when if [absolute] is set) the IP is banned (0 or empty means using the default time of 24h which can also be overwritten by the -bantime startup argument)"}, - {"absolute", RPCArg::Type::BOOL, /* default */ "false", "If set, the bantime must be an absolute timestamp in seconds since epoch (Jan 1 1970 GMT)"}, + {"absolute", RPCArg::Type::BOOL, /* default */ "false", "If set, the bantime must be an absolute timestamp expressed in " + UNIX_EPOCH_TIME}, }, RPCResults{}, RPCExamples{ @@ -546,7 +553,7 @@ static UniValue setban(const JSONRPCRequest& request) if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { throw std::runtime_error(help.ToString()); } - if (!g_banman) { + if (!g_rpc_node->banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } @@ -559,18 +566,18 @@ static UniValue setban(const JSONRPCRequest& request) if (!isSubnet) { CNetAddr resolved; - LookupHost(request.params[0].get_str().c_str(), resolved, false); + LookupHost(request.params[0].get_str(), resolved, false); netAddr = resolved; } else - LookupSubNet(request.params[0].get_str().c_str(), subNet); + LookupSubNet(request.params[0].get_str(), subNet); if (! (isSubnet ? subNet.IsValid() : netAddr.IsValid()) ) throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Invalid IP/Subnet"); if (strCommand == "add") { - if (isSubnet ? g_banman->IsBanned(subNet) : g_banman->IsBanned(netAddr)) { + if (isSubnet ? g_rpc_node->banman->IsBanned(subNet) : g_rpc_node->banman->IsBanned(netAddr)) { throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: IP/Subnet already banned"); } @@ -583,20 +590,20 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - g_banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); - if (g_connman) { - g_connman->DisconnectNode(subNet); + g_rpc_node->banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + if (g_rpc_node->connman) { + g_rpc_node->connman->DisconnectNode(subNet); } } else { - g_banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); - if (g_connman) { - g_connman->DisconnectNode(netAddr); + g_rpc_node->banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + if (g_rpc_node->connman) { + g_rpc_node->connman->DisconnectNode(netAddr); } } } else if(strCommand == "remove") { - if (!( isSubnet ? g_banman->Unban(subNet) : g_banman->Unban(netAddr) )) { + if (!( isSubnet ? g_rpc_node->banman->Unban(subNet) : g_rpc_node->banman->Unban(netAddr) )) { throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); } } @@ -615,12 +622,12 @@ static UniValue listbanned(const JSONRPCRequest& request) }, }.Check(request); - if(!g_banman) { + if(!g_rpc_node->banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } banmap_t banMap; - g_banman->GetBanned(banMap); + g_rpc_node->banman->GetBanned(banMap); UniValue bannedAddresses(UniValue::VARR); for (const auto& entry : banMap) @@ -649,11 +656,11 @@ static UniValue clearbanned(const JSONRPCRequest& request) + HelpExampleRpc("clearbanned", "") }, }.Check(request); - if (!g_banman) { + if (!g_rpc_node->banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } - g_banman->ClearBanned(); + g_rpc_node->banman->ClearBanned(); return NullUniValue; } @@ -669,13 +676,13 @@ static UniValue setnetworkactive(const JSONRPCRequest& request) RPCExamples{""}, }.Check(request); - if (!g_connman) { + if (!g_rpc_node->connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } - g_connman->SetNetworkActive(request.params[0].get_bool()); + g_rpc_node->connman->SetNetworkActive(request.params[0].get_bool()); - return g_connman->GetNetworkActive(); + return g_rpc_node->connman->GetNetworkActive(); } static UniValue getnodeaddresses(const JSONRPCRequest& request) @@ -688,7 +695,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) RPCResult{ "[\n" " {\n" - " \"time\": ttt, (numeric) Timestamp in seconds since epoch (Jan 1 1970 GMT) keeping track of when the node was last seen\n" + " \"time\": ttt, (numeric) The " + UNIX_EPOCH_TIME + " of when the node was last seen\n" " \"services\": n, (numeric) The services offered\n" " \"address\": \"host\", (string) The address of the node\n" " \"port\": n (numeric) The port of the node\n" @@ -701,7 +708,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) + HelpExampleRpc("getnodeaddresses", "8") }, }.Check(request); - if (!g_connman) { + if (!g_rpc_node->connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -713,7 +720,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) } } // returns a shuffled list of CAddress - std::vector<CAddress> vAddr = g_connman->GetAddresses(); + std::vector<CAddress> vAddr = g_rpc_node->connman->GetAddresses(); UniValue ret(UniValue::VARR); int address_return_count = std::min<int>(count, vAddr.size()); diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index ef6537e4ec..d1475f452d 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -63,6 +63,9 @@ enum RPCErrorCode RPC_CLIENT_INVALID_IP_OR_SUBNET = -30, //!< Invalid IP/Subnet RPC_CLIENT_P2P_DISABLED = -31, //!< No valid connection manager instance found + //! Chain errors + RPC_CLIENT_MEMPOOL_DISABLED = -33, //!< No mempool instance found + //! Wallet errors RPC_WALLET_ERROR = -4, //!< Unspecified problem with wallet (key not found etc.) RPC_WALLET_INSUFFICIENT_FUNDS = -6, //!< Not enough funds in wallet or account diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index cdcf0c9971..cea59b2c7a 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -11,6 +11,7 @@ #include <key_io.h> #include <merkleblock.h> #include <node/coin.h> +#include <node/context.h> #include <node/psbt.h> #include <node/transaction.h> #include <policy/policy.h> @@ -18,6 +19,7 @@ #include <primitives/transaction.h> #include <psbt.h> #include <random.h> +#include <rpc/blockchain.h> #include <rpc/rawtransaction_util.h> #include <rpc/server.h> #include <rpc/util.h> @@ -137,7 +139,7 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) " ],\n" " \"blockhash\" : \"hash\", (string) the block hash\n" " \"confirmations\" : n, (numeric) The confirmations\n" - " \"blocktime\" : ttt (numeric) The block time in seconds since epoch (Jan 1 1970 GMT)\n" + " \"blocktime\" : ttt (numeric) The block time expressed in " + UNIX_EPOCH_TIME + "\n" " \"time\" : ttt, (numeric) Same as \"blocktime\"\n" "}\n" }, @@ -634,6 +636,7 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { + const CTxMemPool& mempool = EnsureMemPool(); LOCK(cs_main); LOCK(mempool.cs); CCoinsViewCache &viewChain = ::ChainstateActive().CoinsTip(); @@ -756,12 +759,14 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) for (const CTxIn& txin : mtx.vin) { coins[txin.prevout]; // Create empty map entry keyed by prevout. } - FindCoins(coins); + FindCoins(*g_rpc_node, coins); // Parse the prevtxs array ParsePrevouts(request.params[2], &keystore, coins); - return SignTransaction(mtx, &keystore, coins, request.params[3]); + UniValue result(UniValue::VOBJ); + SignTransaction(mtx, &keystore, coins, request.params[3], result); + return result; } static UniValue sendrawtransaction(const JSONRPCRequest& request) @@ -817,7 +822,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) std::string err_string; AssertLockNotHeld(cs_main); - const TransactionError err = BroadcastTransaction(tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); + const TransactionError err = BroadcastTransaction(*g_rpc_node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); if (TransactionError::OK != err) { throw JSONRPCTransactionError(err, err_string); } @@ -886,6 +891,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); } + CTxMemPool& mempool = EnsureMemPool(); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -893,20 +899,21 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) UniValue result_0(UniValue::VOBJ); result_0.pushKV("txid", tx_hash.GetHex()); - CValidationState state; - bool missing_inputs; + TxValidationState state; bool test_accept_res; { LOCK(cs_main); - test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), &missing_inputs, + test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), nullptr /* plTxnReplaced */, false /* bypass_limits */, max_raw_tx_fee, /* test_accept */ true); } result_0.pushKV("allowed", test_accept_res); if (!test_accept_res) { if (state.IsInvalid()) { - result_0.pushKV("reject-reason", strprintf("%s", state.GetRejectReason())); - } else if (missing_inputs) { - result_0.pushKV("reject-reason", "missing-inputs"); + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + result_0.pushKV("reject-reason", "missing-inputs"); + } else { + result_0.pushKV("reject-reason", strprintf("%s", state.GetRejectReason())); + } } else { result_0.pushKV("reject-reason", state.GetRejectReason()); } @@ -1072,7 +1079,12 @@ UniValue decodepsbt(const JSONRPCRequest& request) UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); - total_in += txout.nValue; + if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { + total_in += txout.nValue; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, o, true); @@ -1082,7 +1094,13 @@ UniValue decodepsbt(const JSONRPCRequest& request) UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); in.pushKV("non_witness_utxo", non_wit); - total_in += input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; + CAmount utxo_val = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; + if (MoneyRange(utxo_val) && MoneyRange(total_in + utxo_val)) { + total_in += utxo_val; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } } else { have_all_utxos = false; } @@ -1198,7 +1216,12 @@ UniValue decodepsbt(const JSONRPCRequest& request) outputs.push_back(out); // Fee calculation - output_value += psbtx.tx->vout[i].nValue; + if (MoneyRange(psbtx.tx->vout[i].nValue) && MoneyRange(output_value + psbtx.tx->vout[i].nValue)) { + output_value += psbtx.tx->vout[i].nValue; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } } result.pushKV("outputs", outputs); if (have_all_utxos) { @@ -1503,6 +1526,7 @@ UniValue utxoupdatepsbt(const JSONRPCRequest& request) CCoinsView viewDummy; CCoinsViewCache view(&viewDummy); { + const CTxMemPool& mempool = EnsureMemPool(); LOCK2(cs_main, mempool.cs); CCoinsViewCache &viewChain = ::ChainstateActive().CoinsTip(); CCoinsViewMemPool viewMempool(&viewChain, mempool); @@ -1602,7 +1626,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) for (auto& psbt : psbtxs) { for (unsigned int i = 0; i < psbt.tx->vin.size(); ++i) { if (!merged_psbt.AddInput(psbt.tx->vin[i], psbt.inputs[i])) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input %s:%d exists in multiple PSBTs", psbt.tx->vin[i].prevout.hash.ToString().c_str(), psbt.tx->vin[i].prevout.n)); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input %s:%d exists in multiple PSBTs", psbt.tx->vin[i].prevout.hash.ToString(), psbt.tx->vin[i].prevout.n)); } } for (unsigned int i = 0; i < psbt.tx->vout.size(); ++i) { @@ -1617,7 +1641,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) std::vector<int> output_indices(merged_psbt.outputs.size()); std::iota(output_indices.begin(), output_indices.end(), 0); - // Shuffle input and output indicies lists + // Shuffle input and output indices lists Shuffle(input_indices.begin(), input_indices.end(), FastRandomContext()); Shuffle(output_indices.begin(), output_indices.end(), FastRandomContext()); @@ -1669,6 +1693,7 @@ UniValue analyzepsbt(const JSONRPCRequest& request) " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction in " + CURRENCY_UNIT + "/kB. Shown only if all UTXO slots in the PSBT have been filled.\n" " \"fee\" : fee (numeric, optional) The transaction fee paid. Shown only if all UTXO slots in the PSBT have been filled.\n" " \"next\" : \"role\" (string) Role of the next person that this psbt needs to go to\n" + " \"error\" : \"error\" (string) Error message if there is one\n" "}\n" }, RPCExamples { @@ -1721,7 +1746,7 @@ UniValue analyzepsbt(const JSONRPCRequest& request) } inputs_result.push_back(input_univ); } - result.pushKV("inputs", inputs_result); + if (!inputs_result.empty()) result.pushKV("inputs", inputs_result); if (psbta.estimated_vsize != nullopt) { result.pushKV("estimated_vsize", (int)*psbta.estimated_vsize); @@ -1733,6 +1758,9 @@ UniValue analyzepsbt(const JSONRPCRequest& request) result.pushKV("fee", ValueFromAmount(*psbta.fee)); } result.pushKV("next", PSBTRoleName(psbta.next)); + if (!psbta.error.empty()) { + result.pushKV("error", psbta.error); + } return result; } diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index fe98fff4bb..40334883c5 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -268,7 +268,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst } } -UniValue SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, const UniValue& hashType) +void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, const UniValue& hashType, UniValue& result) { int nHashType = ParseSighashString(hashType); @@ -319,12 +319,12 @@ UniValue SignTransaction(CMutableTransaction& mtx, const SigningProvider* keysto } bool fComplete = vErrors.empty(); - UniValue result(UniValue::VOBJ); result.pushKV("hex", EncodeHexTx(CTransaction(mtx))); result.pushKV("complete", fComplete); if (!vErrors.empty()) { + if (result.exists("errors")) { + vErrors.push_backV(result["errors"].getValues()); + } result.pushKV("errors", vErrors); } - - return result; } diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 5b92650764..4750fd64ed 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,15 +21,15 @@ class SigningProvider; * @param keystore Temporary keystore containing signing keys * @param coins Map of unspent outputs * @param hashType The signature hash type - * @returns JSON object with details of signed transaction + * @param result JSON object where signed transaction results accumulate */ -UniValue SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, const UniValue& hashType); +void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, const UniValue& hashType, UniValue& result); /** * Parse a prevtxs UniValue array and get the map of coins from it * - * @param prevTxs Array of previous txns outputs that tx depends on but may not yet be in the block chain - * @param keystore A pointer to the temprorary keystore if there is one + * @param prevTxsUnival Array of previous txns outputs that tx depends on but may not yet be in the block chain + * @param keystore A pointer to the temporary keystore if there is one * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call */ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 91d3e1fca4..df8e687d82 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ #include <memory> // for unique_ptr #include <unordered_map> -static CCriticalSection cs_rpcWarmup; +static RecursiveMutex cs_rpcWarmup; static std::atomic<bool> g_rpc_running{false}; static bool fRPCInWarmup GUARDED_BY(cs_rpcWarmup) = true; static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server started"; diff --git a/src/rpc/server.h b/src/rpc/server.h index be9c03bf6b..c91bf1f613 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index adda90c104..78586c22f9 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -13,7 +13,7 @@ #include <tuple> -InitInterfaces* g_rpc_interfaces = nullptr; +const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; void RPCTypeCheck(const UniValue& params, const std::list<UniValueType>& typesExpected, @@ -133,18 +133,18 @@ CPubKey HexToPubKey(const std::string& hex_in) } // Retrieves a public key for an address from the given FillableSigningProvider -CPubKey AddrToPubKey(FillableSigningProvider* const keystore, const std::string& addr_in) +CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& addr_in) { CTxDestination dest = DecodeDestination(addr_in); if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address: " + addr_in); } - CKeyID key = GetKeyForDestination(*keystore, dest); + CKeyID key = GetKeyForDestination(keystore, dest); if (key.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("%s does not refer to a key", addr_in)); } CPubKey vchPubKey; - if (!keystore->GetPubKey(key, vchPubKey)) { + if (!keystore.GetPubKey(key, vchPubKey)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("no full public key for address %s", addr_in)); } if (!vchPubKey.IsFullyValid()) { @@ -430,7 +430,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP std::set<std::string> named_args; for (const auto& arg : m_args) { // Should have unique named arguments - assert(named_args.insert(arg.m_name).second); + CHECK_NONFATAL(named_args.insert(arg.m_name).second); } } @@ -622,11 +622,11 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code - assert(false); + CHECK_NONFATAL(false); // no default case, so the compiler can warn about missing cases } - assert(false); + CHECK_NONFATAL(false); } std::string RPCArg::ToString(const bool oneline) const @@ -663,7 +663,7 @@ std::string RPCArg::ToString(const bool oneline) const // no default case, so the compiler can warn about missing cases } - assert(false); + CHECK_NONFATAL(false); } static std::pair<int64_t, int64_t> ParseRange(const UniValue& value) diff --git a/src/rpc/util.h b/src/rpc/util.h index 72fc7b6286..065a992a88 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -7,29 +7,30 @@ #include <node/transaction.h> #include <outputtype.h> -#include <pubkey.h> #include <protocol.h> +#include <pubkey.h> #include <rpc/protocol.h> #include <rpc/request.h> #include <script/script.h> #include <script/sign.h> #include <script/standard.h> #include <univalue.h> +#include <util/check.h> #include <string> #include <vector> #include <boost/variant.hpp> +/** + * String used to describe UNIX epoch time in documentation, factored out to a + * constant for consistency. + */ +extern const std::string UNIX_EPOCH_TIME; + class FillableSigningProvider; class CPubKey; class CScript; -struct InitInterfaces; - -//! Pointers to interfaces that need to be accessible from RPC methods. Due to -//! limitations of the RPC framework, there's currently no direct way to pass in -//! state to RPC method implementations. -extern InitInterfaces* g_rpc_interfaces; /** Wrapper for UniValue::VType, which includes typeAny: * Used to denote don't care type. */ @@ -74,7 +75,7 @@ extern std::string HelpExampleCli(const std::string& methodname, const std::stri extern std::string HelpExampleRpc(const std::string& methodname, const std::string& args); CPubKey HexToPubKey(const std::string& hex_in); -CPubKey AddrToPubKey(FillableSigningProvider* const keystore, const std::string& addr_in); +CPubKey AddrToPubKey(const FillableSigningProvider& keystore, const std::string& addr_in); CTxDestination AddAndGetMultisigDestination(const int required, const std::vector<CPubKey>& pubkeys, OutputType type, FillableSigningProvider& keystore, CScript& script_out); UniValue DescribeAddress(const CTxDestination& dest); @@ -146,7 +147,7 @@ struct RPCArg { m_oneline_description{oneline_description}, m_type_str{type_str} { - assert(type != Type::ARR && type != Type::OBJ); + CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ); } RPCArg( @@ -165,7 +166,7 @@ struct RPCArg { m_oneline_description{oneline_description}, m_type_str{type_str} { - assert(type == Type::ARR || type == Type::OBJ); + CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ); } bool IsOptional() const; @@ -194,14 +195,14 @@ struct RPCResult { explicit RPCResult(std::string result) : m_cond{}, m_result{std::move(result)} { - assert(!m_result.empty()); + CHECK_NONFATAL(!m_result.empty()); } RPCResult(std::string cond, std::string result) : m_cond{std::move(cond)}, m_result{std::move(result)} { - assert(!m_cond.empty()); - assert(!m_result.empty()); + CHECK_NONFATAL(!m_cond.empty()); + CHECK_NONFATAL(!m_result.empty()); } }; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index fdc859b3a0..927a3f3820 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -41,8 +41,6 @@ void CScheduler::serviceQueue() try { if (!shouldStop() && taskQueue.empty()) { reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); - // Use this chance to get more entropy - RandAddSeedSleep(); } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. diff --git a/src/scheduler.h b/src/scheduler.h index 436f661c59..7080adf34c 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -98,7 +98,7 @@ class SingleThreadedSchedulerClient { private: CScheduler *m_pscheduler; - CCriticalSection m_cs_callbacks_pending; + RecursiveMutex m_cs_callbacks_pending; std::list<std::function<void ()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); bool m_are_callbacks_running GUARDED_BY(m_cs_callbacks_pending) = false; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 536807e1d8..773d6a55c4 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -36,7 +36,7 @@ namespace { // xpubs use other characters too, but already have their own checksum // mechanism. // * Function names like "multi()" use other characters, but mistakes in -// these would generally result in an unparseable descriptor. +// these would generally result in an unparsable descriptor. // * A case error always counts as 1 symbol error. // * Any other 1 character substitution error counts as 1 or 2 symbol errors. // * Any 1 symbol error is always detected. @@ -645,7 +645,7 @@ NODISCARD bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& } uint32_t p; if (!ParseUInt32(std::string(elem.begin(), elem.end()), &p)) { - error = strprintf("Key path value '%s' is not a valid uint32", std::string(elem.begin(), elem.end()).c_str()); + error = strprintf("Key path value '%s' is not a valid uint32", std::string(elem.begin(), elem.end())); return false; } else if (p > 0x7FFFFFFFUL) { error = strprintf("Key path value %u is out of range", p); @@ -783,7 +783,7 @@ std::unique_ptr<DescriptorImpl> ParseScript(Span<const char>& sp, ParseScriptCon uint32_t thres; std::vector<std::unique_ptr<PubkeyProvider>> providers; if (!ParseUInt32(std::string(threshold.begin(), threshold.end()), &thres)) { - error = strprintf("Multi threshold '%s' is not valid", std::string(threshold.begin(), threshold.end()).c_str()); + error = strprintf("Multi threshold '%s' is not valid", std::string(threshold.begin(), threshold.end())); return nullptr; } size_t script_size = 0; @@ -815,8 +815,8 @@ std::unique_ptr<DescriptorImpl> ParseScript(Span<const char>& sp, ParseScriptCon } } if (ctx == ParseScriptContext::P2SH) { - if (script_size + 3 > 520) { - error = strprintf("P2SH script is too large, %d bytes is larger than 520 bytes", script_size + 3); + if (script_size + 3 > MAX_SCRIPT_ELEMENT_SIZE) { + error = strprintf("P2SH script is too large, %d bytes is larger than %d bytes", script_size + 3, MAX_SCRIPT_ELEMENT_SIZE); return nullptr; } } diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 5a1b55259a..a5a41d78dd 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -47,28 +47,28 @@ struct Descriptor { /** Expand a descriptor at a specified position. * - * @param[in] pos: The position at which to expand the descriptor. If IsRange() is false, this is ignored. - * @param[in] provider: The provider to query for private keys in case of hardened derivation. - * @param[out] output_scripts: The expanded scriptPubKeys. - * @param[out] out: Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). - * @param[out] cache: Cache data necessary to evaluate the descriptor at this point without access to private keys. + * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. + * @param[in] provider The provider to query for private keys in case of hardened derivation. + * @param[out] output_scripts The expanded scriptPubKeys. + * @param[out] out Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). + * @param[out] cache Cache data necessary to evaluate the descriptor at this point without access to private keys. */ virtual bool Expand(int pos, const SigningProvider& provider, std::vector<CScript>& output_scripts, FlatSigningProvider& out, std::vector<unsigned char>* cache = nullptr) const = 0; /** Expand a descriptor at a specified position using cached expansion data. * - * @param[in] pos: The position at which to expand the descriptor. If IsRange() is false, this is ignored. - * @param[in] cache: Cached expansion data. - * @param[out] output_scripts: The expanded scriptPubKeys. - * @param[out] out: Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). + * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. + * @param[in] cache Cached expansion data. + * @param[out] output_scripts The expanded scriptPubKeys. + * @param[out] out Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). */ virtual bool ExpandFromCache(int pos, const std::vector<unsigned char>& cache, std::vector<CScript>& output_scripts, FlatSigningProvider& out) const = 0; /** Expand the private key for a descriptor at a specified position, if possible. * - * @param[in] pos: The position at which to expand the descriptor. If IsRange() is false, this is ignored. - * @param[in] provider: The provider to query for the private keys. - * @param[out] out: Any private keys available for the specified `pos`. + * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. + * @param[in] provider The provider to query for the private keys. + * @param[out] out Any private keys available for the specified `pos`. */ virtual void ExpandPrivate(int pos, const SigningProvider& provider, FlatSigningProvider& out) const = 0; }; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 20fae2eebf..d0865d2793 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -61,17 +61,17 @@ static inline void popstack(std::vector<valtype>& stack) } bool static IsCompressedOrUncompressedPubKey(const valtype &vchPubKey) { - if (vchPubKey.size() < CPubKey::COMPRESSED_PUBLIC_KEY_SIZE) { + if (vchPubKey.size() < CPubKey::COMPRESSED_SIZE) { // Non-canonical public key: too short return false; } if (vchPubKey[0] == 0x04) { - if (vchPubKey.size() != CPubKey::PUBLIC_KEY_SIZE) { + if (vchPubKey.size() != CPubKey::SIZE) { // Non-canonical public key: invalid length for uncompressed key return false; } } else if (vchPubKey[0] == 0x02 || vchPubKey[0] == 0x03) { - if (vchPubKey.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE) { + if (vchPubKey.size() != CPubKey::COMPRESSED_SIZE) { // Non-canonical public key: invalid length for compressed key return false; } @@ -83,7 +83,7 @@ bool static IsCompressedOrUncompressedPubKey(const valtype &vchPubKey) { } bool static IsCompressedPubKey(const valtype &vchPubKey) { - if (vchPubKey.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE) { + if (vchPubKey.size() != CPubKey::COMPRESSED_SIZE) { // Non-canonical public key: invalid length for compressed key return false; } @@ -1281,13 +1281,11 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn return ss.GetHash(); } - static const uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); - // Check for invalid use of SIGHASH_SINGLE if ((nHashType & 0x1f) == SIGHASH_SINGLE) { if (nIn >= txTo.vout.size()) { // nOut out of range - return one; + return UINT256_ONE(); } } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index d63d8b85b7..2b104a608c 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/script.cpp b/src/script/script.cpp index 0666a385d1..ae0de1d24e 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/script.h b/src/script/script.h index 6355b8a704..7aaa10b60b 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index eaf5363bd7..e7b6df3ce8 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 0ed92e8d5b..58eae3ce96 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -144,8 +144,13 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(valtype()); // workaround CHECKMULTISIG bug for (size_t i = 1; i < vSolutions.size() - 1; ++i) { CPubKey pubkey = CPubKey(vSolutions[i]); - if (ret.size() < required + 1 && CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) { - ret.push_back(std::move(sig)); + // We need to always call CreateSig in order to fill sigdata with all + // possible signatures that we can create. This will allow further PSBT + // processing to work as it needs all possible signature and pubkey pairs + if (CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) { + if (ret.size() < required + 1) { + ret.push_back(std::move(sig)); + } } } bool ok = ret.size() == required + 1; diff --git a/src/script/sign.h b/src/script/sign.h index 9d0a5b4d70..033c9ba19e 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -101,7 +101,7 @@ template<typename Stream> void DeserializeHDKeypaths(Stream& s, const std::vector<unsigned char>& key, std::map<CPubKey, KeyOriginInfo>& hd_keypaths) { // Make sure that the key is the size of pubkey + 1 - if (key.size() != CPubKey::PUBLIC_KEY_SIZE + 1 && key.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1) { + if (key.size() != CPubKey::SIZE + 1 && key.size() != CPubKey::COMPRESSED_SIZE + 1) { throw std::ios_base::failure("Size of key was not the expected size for the type BIP32 keypath"); } // Read in the pubkey from key diff --git a/src/script/signingprovider.h b/src/script/signingprovider.h index 4eec2311d4..6ad20480a7 100644 --- a/src/script/signingprovider.h +++ b/src/script/signingprovider.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -63,8 +63,6 @@ FlatSigningProvider Merge(const FlatSigningProvider& a, const FlatSigningProvide class FillableSigningProvider : public SigningProvider { protected: - mutable CCriticalSection cs_KeyStore; - using KeyMap = std::map<CKeyID, CKey>; using ScriptMap = std::map<CScriptID, CScript>; @@ -74,6 +72,8 @@ protected: void ImplicitlyLearnRelatedKeyScripts(const CPubKey& pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); public: + mutable RecursiveMutex cs_KeyStore; + virtual bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey); virtual bool AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } virtual bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index fc6898f444..7d89a336fb 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -44,12 +44,12 @@ const char* GetTxnOutputType(txnouttype t) static bool MatchPayToPubkey(const CScript& script, valtype& pubkey) { - if (script.size() == CPubKey::PUBLIC_KEY_SIZE + 2 && script[0] == CPubKey::PUBLIC_KEY_SIZE && script.back() == OP_CHECKSIG) { - pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::PUBLIC_KEY_SIZE + 1); + if (script.size() == CPubKey::SIZE + 2 && script[0] == CPubKey::SIZE && script.back() == OP_CHECKSIG) { + pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::SIZE + 1); return CPubKey::ValidSize(pubkey); } - if (script.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 2 && script[0] == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE && script.back() == OP_CHECKSIG) { - pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1); + if (script.size() == CPubKey::COMPRESSED_SIZE + 2 && script[0] == CPubKey::COMPRESSED_SIZE && script.back() == OP_CHECKSIG) { + pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::COMPRESSED_SIZE + 1); return CPubKey::ValidSize(pubkey); } return false; diff --git a/src/script/standard.h b/src/script/standard.h index 6db28dbc2d..49a45f3eba 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -47,7 +47,7 @@ extern unsigned nMaxDatacarrierBytes; * but in the future other flags may be added, such as a soft-fork to enforce * strict DER encoding. * - * Failing one of these tests may trigger a DoS ban - see CheckInputs() for + * Failing one of these tests may trigger a DoS ban - see CheckInputScripts() for * details. */ static const unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS = SCRIPT_VERIFY_P2SH; @@ -80,9 +80,14 @@ struct PKHash : public uint160 using uint160::uint160; }; +struct WitnessV0KeyHash; struct ScriptHash : public uint160 { ScriptHash() : uint160() {} + // These don't do what you'd expect. + // Use ScriptHash(GetScriptForDestination(...)) instead. + explicit ScriptHash(const WitnessV0KeyHash& hash) = delete; + explicit ScriptHash(const PKHash& hash) = delete; explicit ScriptHash(const uint160& hash) : uint160(hash) {} explicit ScriptHash(const CScript& script); using uint160::uint160; diff --git a/src/serialize.h b/src/serialize.h index ef270dbbe3..7fa669ebdb 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -199,6 +199,52 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } SerializationOp(s, CSerActionUnserialize()); \ } +/** + * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). + * + * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic + * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when + * deserializing). + * + * Example use: + * struct FooFormatter { + * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } + * } + * would define a class FooFormatter that defines a serialization of Class objects consisting + * of serializing its val1 member using the default serialization, and its val2 member using + * VARINT serialization. That FooFormatter can then be used in statements like + * READWRITE(Using<FooFormatter>(obj.bla)). + */ +#define FORMATTER_METHODS(cls, obj) \ + template<typename Stream> \ + static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \ + template<typename Stream> \ + static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \ + template<typename Stream, typename Type, typename Operation> \ + static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ + +/** + * Implement the Serialize and Unserialize methods by delegating to a single templated + * static method that takes the to-be-(de)serialized object as a parameter. This approach + * has the advantage that the constness of the object becomes a template parameter, and + * thus allows a single implementation that sees the object as const for serializing + * and non-const for deserializing, without casts. + */ +#define SERIALIZE_METHODS(cls, obj) \ + template<typename Stream> \ + void Serialize(Stream& s) const \ + { \ + static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \ + Ser(s, *this); \ + } \ + template<typename Stream> \ + void Unserialize(Stream& s) \ + { \ + static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \ + Unser(s, *this); \ + } \ + FORMATTER_METHODS(cls, obj) + #ifndef CHAR_EQUALS_INT8 template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char #endif @@ -418,26 +464,48 @@ I ReadVarInt(Stream& is) } } -#define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) -#define COMPACTSIZE(obj) CCompactSize(REF(obj)) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) - -template<VarIntMode Mode, typename I> -class CVarInt +/** Simple wrapper class to serialize objects using a formatter; used by Using(). */ +template<typename Formatter, typename T> +class Wrapper { + static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T"); protected: - I &n; + T m_object; public: - explicit CVarInt(I& nIn) : n(nIn) { } + explicit Wrapper(T obj) : m_object(obj) {} + template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); } + template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); } +}; - template<typename Stream> - void Serialize(Stream &s) const { - WriteVarInt<Stream,Mode,I>(s, n); +/** Cause serialization/deserialization of an object to be done using a specified formatter class. + * + * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for + * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside + * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object). + * + * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is + * const during serialization, and non-const during deserialization, which maintains const + * correctness. + */ +template<typename Formatter, typename T> +static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); } + +#define VARINT(obj, ...) Using<VarIntFormatter<__VA_ARGS__>>(obj) +#define COMPACTSIZE(obj) CCompactSize(REF(obj)) +#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) + +/** Serialization wrapper class for integers in VarInt format. */ +template<VarIntMode Mode=VarIntMode::DEFAULT> +struct VarIntFormatter +{ + template<typename Stream, typename I> void Ser(Stream &s, I v) + { + WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v); } - template<typename Stream> - void Unserialize(Stream& s) { - n = ReadVarInt<Stream,Mode,I>(s); + template<typename Stream, typename I> void Unser(Stream& s, I& v) + { + v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s); } }; @@ -522,9 +590,6 @@ public: } }; -template<VarIntMode Mode=VarIntMode::DEFAULT, typename I> -CVarInt<Mode, I> WrapVarInt(I& n) { return CVarInt<Mode, I>{n}; } - template<typename I> BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); } diff --git a/src/streams.h b/src/streams.h index b598dc1aeb..e1d1b0eab2 100644 --- a/src/streams.h +++ b/src/streams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 57f5b1f733..0e31ad3ce3 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/cleanse.cpp b/src/support/cleanse.cpp index ecb00510f7..a8ddcd793f 100644 --- a/src/support/cleanse.cpp +++ b/src/support/cleanse.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/cleanse.h b/src/support/cleanse.h index b03520315d..8c1210a114 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 5c2050e4a2..6980b6c0da 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,10 @@ #endif #include <algorithm> +#ifdef ARENA_DEBUG +#include <iomanip> +#include <iostream> +#endif LockedPoolManager* LockedPoolManager::_instance = nullptr; std::once_flag LockedPoolManager::init_flag; @@ -137,7 +141,7 @@ Arena::Stats Arena::stats() const } #ifdef ARENA_DEBUG -static void printchunk(char* base, size_t sz, bool used) { +static void printchunk(void* base, size_t sz, bool used) { std::cout << "0x" << std::hex << std::setw(16) << std::setfill('0') << base << " 0x" << std::hex << std::setw(16) << std::setfill('0') << sz << @@ -149,7 +153,7 @@ void Arena::walk() const printchunk(chunk.first, chunk.second, true); std::cout << std::endl; for (const auto& chunk: chunks_free) - printchunk(chunk.first, chunk.second, false); + printchunk(chunk.first, chunk.second->first, false); std::cout << std::endl; } #endif diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index b420c909fc..de668f0773 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.cpp b/src/sync.cpp index 653800ae4e..924e7b5bb0 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,7 +75,7 @@ typedef std::set<std::pair<void*, void*> > InvLockOrders; struct LockData { // Very ugly hack: as the global constructs and destructors run single // threaded, we use this boolean to know whether LockData still exists, - // as DeleteLock can get called by global CCriticalSection destructors + // as DeleteLock can get called by global RecursiveMutex destructors // after LockData disappears. bool available; LockData() : available(true) {} @@ -173,7 +173,7 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, for (const std::pair<void*, CLockLocation>& i : g_lockstack) if (i.first == cs) return; - tfm::format(std::cerr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + tfm::format(std::cerr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } @@ -181,7 +181,7 @@ void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLi { for (const std::pair<void*, CLockLocation>& i : g_lockstack) { if (i.first == cs) { - tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } } diff --git a/src/sync.h b/src/sync.h index 8ff6173142..0cdbb59c70 100644 --- a/src/sync.h +++ b/src/sync.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -106,7 +106,6 @@ public: * TODO: We should move away from using the recursive lock by default. */ using RecursiveMutex = AnnotatedMixin<std::recursive_mutex>; -typedef AnnotatedMixin<std::recursive_mutex> CCriticalSection; /** Wrapped mutex: supports waiting but not recursive locking */ typedef AnnotatedMixin<std::mutex> Mutex; diff --git a/src/test/README.md b/src/test/README.md index 96dcb072bc..731720f654 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -7,8 +7,8 @@ configure some other framework (we want as few impediments to creating unit tests as possible). The build system is set up to compile an executable called `test_bitcoin` -that runs all of the unit tests. The main source file is called -`setup_common.cpp`. +that runs all of the unit tests. The main source file for the test library is found in +`util/setup_common.cpp`. ### Compiling/running unit tests diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index da0abd495a..07cebeb35a 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -2,9 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> +#include <util/asmap.h> +#include <test/data/asmap.raw.h> #include <hash.h> #include <netbase.h> @@ -12,13 +14,18 @@ class CAddrManTest : public CAddrMan { +private: + bool deterministic; public: - explicit CAddrManTest(bool makeDeterministic = true) + explicit CAddrManTest(bool makeDeterministic = true, + std::vector<bool> asmap = std::vector<bool>()) { if (makeDeterministic) { // Set addrman addr placement to be deterministic. MakeDeterministic(); } + deterministic = makeDeterministic; + m_asmap = asmap; } //! Ensure that bucket placement is always the same for testing purposes. @@ -46,6 +53,21 @@ public: CAddrMan::Delete(nId); } + // Used to test deserialization + std::pair<int, int> GetBucketAndEntry(const CAddress& addr) + { + LOCK(cs); + int nId = mapAddr[addr]; + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { + if (nId == vvNew[bucket][entry]) { + return std::pair<int, int>(bucket, entry); + } + } + } + return std::pair<int, int>(-1, -1); + } + // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(CService& addr) { @@ -57,32 +79,45 @@ public: int64_t nLastTry = GetAdjustedTime()-61; Attempt(addr, count_failure, nLastTry); } + + void Clear() + { + CAddrMan::Clear(); + if (deterministic) { + nKey.SetNull(); + insecure_rand = FastRandomContext(true); + } + } + }; -static CNetAddr ResolveIP(const char* ip) +static CNetAddr ResolveIP(const std::string& ip) { CNetAddr addr; BOOST_CHECK_MESSAGE(LookupHost(ip, addr, false), strprintf("failed to resolve: %s", ip)); return addr; } -static CNetAddr ResolveIP(std::string ip) -{ - return ResolveIP(ip.c_str()); -} - -static CService ResolveService(const char* ip, int port = 0) +static CService ResolveService(const std::string& ip, const int port = 0) { CService serv; BOOST_CHECK_MESSAGE(Lookup(ip, serv, port, false), strprintf("failed to resolve: %s:%i", ip, port)); return serv; } -static CService ResolveService(std::string ip, int port = 0) -{ - return ResolveService(ip.c_str(), port); + +static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) { + std::vector<bool> result(vector_size); + for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { + unsigned char cur_byte = source[byte_i]; + for (int bit_i = 0; bit_i < 8; ++bit_i) { + result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1; + } + } + return result; } + BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) @@ -409,7 +444,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } -BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { CAddrManTest addrman; @@ -424,30 +459,31 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info2.GetTriedBucket(nKey1)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetTriedBucket(nKey1); + int bucket = infoi.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same group (\16 prefix for IPv4) should - // never get more than 8 buckets + // Test: IP addresses in the same /16 prefix should + // never get more than 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 8U); buckets.clear(); @@ -455,15 +491,15 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), ResolveIP("250." + std::to_string(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1); + int bucket = infoj.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different groups should map to more than - // 8 buckets. + // Test: IP addresses in the different /16 prefix should map to more than + // 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 160U); } -BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { CAddrManTest addrman; @@ -477,25 +513,27 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 + // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetNewBucket(nKey1); + int bucket = infoi.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } // Test: IP addresses in the same group (\16 prefix for IPv4) should @@ -508,10 +546,10 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to NO MORE // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -520,14 +558,226 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + std::to_string(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to MORE // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } +// The following three test cases use asmap.raw +// We use an artificial minimal mock mapping +// 250.0.0.0/8 AS1000 +// 101.1.0.0/16 AS1 +// 101.2.0.0/16 AS2 +// 101.3.0.0/16 AS3 +// 101.4.0.0/16 AS4 +// 101.5.0.0/16 AS5 +// 101.6.0.0/16 AS6 +// 101.7.0.0/16 AS7 +// 101.8.0.0/16 AS8 +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.1.1"); + + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + + // Test: Two addresses with same IP but different ports can map to + // different buckets because they have different keys. + CAddrInfo info2 = CAddrInfo(addr2, source1); + + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("101." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() > 8); + + buckets.clear(); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY NOT map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() == 8); +} + +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.2.1"); + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + + // Test: Ports should not affect bucket placement in the addr + CAddrInfo info2 = CAddrInfo(addr2, source1); + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same /16 prefix + // usually map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); + + buckets.clear(); + for (int j = 0; j < 4 * 255; j++) { + CAddrInfo infoj = CAddrInfo(CAddress( + ResolveService( + std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), + ResolveIP("251.4.1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same source /16 prefix should not map to more + // than 64 buckets. + BOOST_CHECK(buckets.size() <= 64); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes usually map to MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() > 1); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() == 1); + +} + +BOOST_AUTO_TEST_CASE(addrman_serialization) +{ + std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + CAddrManTest addrman_asmap1(true, asmap1); + CAddrManTest addrman_asmap1_dup(true, asmap1); + CAddrManTest addrman_noasmap; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CNetAddr default_source; + + + addrman_asmap1.Add(addr, default_source); + + stream << addrman_asmap1; + // serizalizing/deserializing addrman with the same asmap + stream >> addrman_asmap1_dup; + + std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); + std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + + BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + + // deserializing asmaped peers.dat to non-asmaped addrman + stream << addrman_asmap1; + stream >> addrman_noasmap; + std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_noasmap.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + + // deserializing non-asmaped peers.dat to asmaped addrman + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + addrman_noasmap.Add(addr, default_source); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + // used to map to different buckets, now maps to the same bucket. + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); + addrman_noasmap.Add(addr, default_source); + addrman_noasmap.Add(addr2, default_source); + std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); +} + BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index e333763f27..d33d668a04 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -5,7 +5,7 @@ #include <util/memory.h> #include <util/system.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <memory> diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 378fe285d5..e20900ed13 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -4,7 +4,7 @@ #include <amount.h> #include <policy/feerate.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 9ac87261b6..3723a48903 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -11,7 +11,7 @@ #include <uint256.h> #include <arith_uint256.h> #include <string> -#include <test/setup_common.h> +#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index b3bed2434c..690368b177 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -20,6 +20,17 @@ BOOST_AUTO_TEST_CASE(base32_testvectors) std::string strDec = DecodeBase32(vstrOut[i]); BOOST_CHECK_EQUAL(strDec, vstrIn[i]); } + + // Decoding strings with embedded NUL characters should fail + bool failure; + (void)DecodeBase32(std::string("invalid", 7), &failure); + BOOST_CHECK_EQUAL(failure, true); + (void)DecodeBase32(std::string("AWSX3VPP", 8), &failure); + BOOST_CHECK_EQUAL(failure, false); + (void)DecodeBase32(std::string("AWSX3VPP\0invalid", 16), &failure); + BOOST_CHECK_EQUAL(failure, true); + (void)DecodeBase32(std::string("AWSX3VPPinvalid", 15), &failure); + BOOST_CHECK_EQUAL(failure, true); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index cb376cddb6..57559fa687 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -5,8 +5,9 @@ #include <test/data/base58_encode_decode.json.h> #include <base58.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> +#include <util/vector.h> #include <univalue.h> @@ -53,17 +54,45 @@ BOOST_AUTO_TEST_CASE(base58_DecodeBase58) } std::vector<unsigned char> expected = ParseHex(test[0].get_str()); std::string base58string = test[1].get_str(); - BOOST_CHECK_MESSAGE(DecodeBase58(base58string, result), strTest); + BOOST_CHECK_MESSAGE(DecodeBase58(base58string, result, 256), strTest); BOOST_CHECK_MESSAGE(result.size() == expected.size() && std::equal(result.begin(), result.end(), expected.begin()), strTest); } - BOOST_CHECK(!DecodeBase58("invalid", result)); + BOOST_CHECK(!DecodeBase58("invalid", result, 100)); + BOOST_CHECK(!DecodeBase58(std::string("invalid"), result, 100)); + BOOST_CHECK(!DecodeBase58(std::string("\0invalid", 8), result, 100)); + + BOOST_CHECK(DecodeBase58(std::string("good", 4), result, 100)); + BOOST_CHECK(!DecodeBase58(std::string("bad0IOl", 7), result, 100)); + BOOST_CHECK(!DecodeBase58(std::string("goodbad0IOl", 11), result, 100)); + BOOST_CHECK(!DecodeBase58(std::string("good\0bad0IOl", 12), result, 100)); // check that DecodeBase58 skips whitespace, but still fails with unexpected non-whitespace at the end. - BOOST_CHECK(!DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t a", result)); - BOOST_CHECK( DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t ", result)); + BOOST_CHECK(!DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t a", result, 3)); + BOOST_CHECK( DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t ", result, 3)); std::vector<unsigned char> expected = ParseHex("971a55"); BOOST_CHECK_EQUAL_COLLECTIONS(result.begin(), result.end(), expected.begin(), expected.end()); + + BOOST_CHECK(DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh", 21), result, 100)); + BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oi", 21), result, 100)); + BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh0IOl", 25), result, 100)); + BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh\00IOl", 26), result, 100)); +} + +BOOST_AUTO_TEST_CASE(base58_random_encode_decode) +{ + for (int n = 0; n < 1000; ++n) { + unsigned int len = 1 + InsecureRandBits(8); + unsigned int zeroes = InsecureRandBool() ? InsecureRandRange(len + 1) : 0; + auto data = Cat(std::vector<unsigned char>(zeroes, '\000'), g_insecure_rand_ctx.randbytes(len - zeroes)); + auto encoded = EncodeBase58Check(data); + std::vector<unsigned char> decoded; + auto ok_too_small = DecodeBase58Check(encoded, decoded, InsecureRandRange(len)); + BOOST_CHECK(!ok_too_small); + auto ok = DecodeBase58Check(encoded, decoded, len + InsecureRandRange(257 - len)); + BOOST_CHECK(ok); + BOOST_CHECK(data == decoded); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 9ffffb0b7d..94df4d1955 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -20,6 +20,17 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) std::string strDec = DecodeBase64(strEnc); BOOST_CHECK_EQUAL(strDec, vstrIn[i]); } + + // Decoding strings with embedded NUL characters should fail + bool failure; + (void)DecodeBase64(std::string("invalid", 7), &failure); + BOOST_CHECK_EQUAL(failure, true); + (void)DecodeBase64(std::string("nQB/pZw=", 8), &failure); + BOOST_CHECK_EQUAL(failure, false); + (void)DecodeBase64(std::string("nQB/pZw=\0invalid", 16), &failure); + BOOST_CHECK_EQUAL(failure, true); + (void)DecodeBase64(std::string("nQB/pZw=invalid", 15), &failure); + BOOST_CHECK_EQUAL(failure, true); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index 0ba492c24e..a2098f4f56 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -3,25 +3,13 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bech32.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> +#include <test/util/str.h> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(bech32_tests, BasicTestingSetup) -static bool CaseInsensitiveEqual(const std::string &s1, const std::string &s2) -{ - if (s1.size() != s2.size()) return false; - for (size_t i = 0; i < s1.size(); ++i) { - char c1 = s1[i]; - if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); - char c2 = s2[i]; - if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); - if (c1 != c2) return false; - } - return true; -} - BOOST_AUTO_TEST_CASE(bip173_testvectors_valid) { static const std::string CASES[] = { diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index e46cf624cf..53df032252 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -9,7 +9,7 @@ #include <key_io.h> #include <streams.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <vector> diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index ca75563ef0..3b4c480f72 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -1,10 +1,14 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <boost/test/unit_test.hpp> #include <stdlib.h> #include <chain.h> #include <rpc/blockchain.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 5ce8e6feb0..8694891a51 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -3,22 +3,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockencodings.h> -#include <consensus/merkle.h> #include <chainparams.h> +#include <consensus/merkle.h> #include <pow.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> std::vector<std::pair<uint256, CTransactionRef>> extra_txn; -struct RegtestingSetup : public TestingSetup { - RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} -}; - -BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegtestingSetup) +BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegTestingSetup) static CBlock BuildBlockTestCase() { CBlock block; diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index ba293b7836..79e18cd2c0 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -8,8 +8,9 @@ #include <index/blockfilterindex.h> #include <miner.h> #include <pow.h> -#include <test/setup_common.h> #include <script/standard.h> +#include <test/util/blockfilter.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <validation.h> @@ -17,22 +18,10 @@ BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) -static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, - BlockFilter& filter) -{ - CBlock block; - if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { - return false; - } - - CBlockUndo block_undo; - if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { - return false; - } - - filter = BlockFilter(filter_type, block, block_undo); - return true; -} +struct BuildChainTestingSetup : public TestChain100Setup { + CBlock CreateBlock(const CBlockIndex* prev, const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey); + bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, size_t length, std::vector<std::shared_ptr<CBlock>>& chain); +}; static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, uint256& last_header) @@ -68,12 +57,12 @@ static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex return true; } -static CBlock CreateBlock(const CBlockIndex* prev, - const std::vector<CMutableTransaction>& txns, - const CScript& scriptPubKey) +CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; block.hashPrevBlock = prev->GetBlockHash(); block.nTime = prev->nTime + 1; @@ -92,8 +81,10 @@ static CBlock CreateBlock(const CBlockIndex* prev, return block; } -static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, - size_t length, std::vector<std::shared_ptr<CBlock>>& chain) +bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, + const CScript& coinbase_script_pub_key, + size_t length, + std::vector<std::shared_ptr<CBlock>>& chain) { std::vector<CMutableTransaction> no_txns; @@ -102,8 +93,8 @@ static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key)); CBlockHeader header = block->GetBlockHeader(); - CValidationState state; - if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex, nullptr)) { + BlockValidationState state; + if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } @@ -111,7 +102,7 @@ static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script return true; } -BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true); diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index df0a041e0e..e69503ef35 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/data/blockfilters.json.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <blockfilter.h> #include <core_io.h> diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 4421494007..4a7ad9b38b 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -15,7 +15,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -461,7 +461,7 @@ static std::vector<unsigned char> RandomData() BOOST_AUTO_TEST_CASE(rolling_bloom) { - SeedInsecureRand(/* deterministic */ true); + SeedInsecureRand(SeedRand::ZEROS); g_mock_deterministic_tests = true; // last-100-entry, 1% false positive: diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 8fd4e5d5d6..d5e2344a8b 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compat/byteswap.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index d796444419..482fe3772c 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -5,9 +5,8 @@ #include <util/memory.h> #include <util/system.h> #include <util/time.h> -#include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> @@ -18,12 +17,12 @@ #include <condition_variable> #include <unordered_set> +#include <utility> -// BasicTestingSetup not sufficient because nScriptCheckThreads is not set -// otherwise. BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) static const unsigned int QUEUE_BATCH_SIZE = 128; +static const int SCRIPT_CHECK_THREADS = 3; struct FakeCheck { bool operator()() @@ -149,7 +148,7 @@ static void Correct_Queue_range(std::vector<size_t> range) { auto small_queue = MakeUnique<Correct_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{small_queue->Thread();}); } // Make vChecks here to save on malloc (this test can be slow...) @@ -214,7 +213,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{fail_queue->Thread();}); } @@ -246,7 +245,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{fail_queue->Thread();}); } @@ -274,7 +273,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) { auto queue = MakeUnique<Unique_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{queue->Thread();}); } @@ -310,7 +309,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) { auto queue = MakeUnique<Memory_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{queue->Thread();}); } for (size_t i = 0; i < 1000; ++i) { @@ -342,7 +341,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) auto queue = MakeUnique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; bool fails = false; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{queue->Thread();}); } std::thread t0([&]() { diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 948591196c..436c1bffa0 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -7,7 +7,7 @@ #include <coins.h> #include <script/standard.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <uint256.h> #include <undo.h> #include <util/strencodings.h> @@ -279,7 +279,7 @@ UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { // has the expected effect (the other duplicate is overwritten at all cache levels) BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { - SeedInsecureRand(/* deterministic */ true); + SeedInsecureRand(SeedRand::ZEROS); g_mock_deterministic_tests = true; bool spent_a_duplicate_coinbase = false; diff --git a/src/test/compilerbug_tests.cpp b/src/test/compilerbug_tests.cpp index 74e1eac3ea..1a6fcda009 100644 --- a/src/test/compilerbug_tests.cpp +++ b/src/test/compilerbug_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(compilerbug_tests, BasicTestingSetup) diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index c6a08b293f..22eae91cf0 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <script/standard.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4ac12bf969..2deb0c5bfc 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -15,7 +15,7 @@ #include <crypto/sha512.h> #include <random.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -186,14 +186,15 @@ static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &sa BOOST_CHECK(HexStr(out, out + 32) == okm_check_hex); } -static std::string LongTestString() { +static std::string LongTestString() +{ std::string ret; - for (int i=0; i<200000; i++) { - ret += (unsigned char)(i); - ret += (unsigned char)(i >> 4); - ret += (unsigned char)(i >> 8); - ret += (unsigned char)(i >> 12); - ret += (unsigned char)(i >> 16); + for (int i = 0; i < 200000; i++) { + ret += (char)(i); + ret += (char)(i >> 4); + ret += (char)(i >> 8); + ret += (char)(i >> 12); + ret += (char)(i >> 16); } return ret; } diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index d38ede691a..6be24c0845 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -4,17 +4,18 @@ #include <boost/test/unit_test.hpp> #include <cuckoocache.h> #include <script/sigcache.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <random.h> #include <thread> +#include <deque> /** Test Suite for CuckooCache * - * 1) All tests should have a deterministic result (using insecure rand + * 1. All tests should have a deterministic result (using insecure rand * with deterministic seeds) - * 2) Some test methods are templated to allow for easier testing + * 2. Some test methods are templated to allow for easier testing * against new versions / comparing - * 3) Results should be treated as a regression test, i.e., did the behavior + * 3. Results should be treated as a regression test, i.e., did the behavior * change significantly from what was expected. This can be OK, depending on * the nature of the change, but requires updating the tests to reflect the new * expected behavior. For example improving the hit rate may cause some tests @@ -29,7 +30,7 @@ BOOST_AUTO_TEST_SUITE(cuckoocache_tests); */ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); @@ -47,7 +48,7 @@ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) template <typename Cache> static double test_cache(size_t megabytes, double load) { - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -82,9 +83,9 @@ static double test_cache(size_t megabytes, double load) * * Examples: * - * 1) at load 0.5, we expect a perfect hit rate, so we multiply by + * 1. at load 0.5, we expect a perfect hit rate, so we multiply by * 1.0 - * 2) at load 2.0, we expect to see half the entries, so a perfect hit rate + * 2. at load 2.0, we expect to see half the entries, so a perfect hit rate * would be 0.5. Therefore, if we see a hit rate of 0.4, 0.4*2.0 = 0.8 is the * normalized hit rate. * @@ -118,7 +119,7 @@ template <typename Cache> static void test_cache_erase(size_t megabytes) { double load = 1; - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -181,7 +182,7 @@ template <typename Cache> static void test_cache_erase_parallel(size_t megabytes) { double load = 1; - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -285,7 +286,7 @@ static void test_cache_generations() // iterations with non-deterministic values, so it isn't "overfit" to the // specific entropy in FastRandomContext(true) and implementation of the // cache. - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); // block_activity models a chunk of network activity. n_insert elements are // added to the cache. The first and last n/4 are stored for removal later diff --git a/src/test/data/asmap.raw b/src/test/data/asmap.raw Binary files differnew file mode 100644 index 0000000000..3dcf1f3940 --- /dev/null +++ b/src/test/data/asmap.raw diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 2ffe4dccdb..b647c0f70b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -4,7 +4,7 @@ #include <dbwrapper.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/memory.h> #include <memory> @@ -397,6 +397,18 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) } } +BOOST_AUTO_TEST_CASE(unicodepath) +{ + // Attempt to create a database with a utf8 character in the path. + // On Windows this test will fail if the directory is created using + // the ANSI CreateDirectoryA call and the code page isn't UTF8. + // It will succeed if the created with CreateDirectoryW. + fs::path ph = GetDataDir() / "test_runner_₿_🏃_20191128_104644"; + CDBWrapper dbw(ph, (1 << 20)); + + fs::path lockPath = ph / "LOCK"; + BOOST_CHECK(boost::filesystem::exists(lockPath)); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 1928324b27..2c2b3035e3 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,7 +17,7 @@ #include <util/time.h> #include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <stdint.h> @@ -51,7 +51,7 @@ struct COrphanTx { NodeId fromPeer; int64_t nTimeExpire; }; -extern CCriticalSection g_cs_orphans; +extern RecursiveMutex g_cs_orphans; extern std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); static CService ip(uint32_t i) diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 55726a4a8f..bcce8854e3 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -6,7 +6,7 @@ #include <string> #include <script/sign.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <script/descriptor.h> #include <util/strencodings.h> diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index 740d805cce..9bb0b3ef02 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -5,7 +5,7 @@ #include <clientversion.h> #include <flatfile.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index b504a3cbb1..d02c3613ba 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <fs.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/fuzz/FuzzedDataProvider.h b/src/test/fuzz/FuzzedDataProvider.h index 1b5b4bb012..3e069eba69 100644 --- a/src/test/fuzz/FuzzedDataProvider.h +++ b/src/test/fuzz/FuzzedDataProvider.h @@ -13,11 +13,10 @@ #ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ #define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ -#include <limits.h> -#include <stddef.h> -#include <stdint.h> - #include <algorithm> +#include <climits> +#include <cstddef> +#include <cstdint> #include <cstring> #include <initializer_list> #include <string> @@ -25,8 +24,10 @@ #include <utility> #include <vector> +// In addition to the comments below, the API is also briefly documented at +// https://github.com/google/fuzzing/blob/master/docs/split-inputs.md#fuzzed-data-provider class FuzzedDataProvider { -public: + public: // |data| is an array of length |size| that the FuzzedDataProvider wraps to // provide more granular access. |data| must outlive the FuzzedDataProvider. FuzzedDataProvider(const uint8_t *data, size_t size) @@ -143,9 +144,9 @@ public: return ConsumeBytes<T>(remaining_bytes_); } + // Returns a std::string containing all remaining bytes of the input data. // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string // object. - // Returns a std::vector containing all remaining bytes of the input data. std::string ConsumeRemainingBytesAsString() { return ConsumeBytesAsString(remaining_bytes_); } @@ -161,7 +162,7 @@ public: // Reads one byte and returns a bool, or false when no data remains. bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); } - // Returns a copy of a value selected from a fixed-size |array|. + // Returns a copy of the value selected from the given fixed-size |array|. template <typename T, size_t size> T PickValueInArray(const T (&array)[size]) { static_assert(size > 0, "The array must be non empty."); @@ -170,11 +171,14 @@ public: template <typename T> T PickValueInArray(std::initializer_list<const T> list) { - // static_assert(list.size() > 0, "The array must be non empty."); + // TODO(Dor1s): switch to static_assert once C++14 is allowed. + if (!list.size()) + abort(); + return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1)); } - // Return an enum value. The enum must start at 0 and be contiguous. It must + // Returns an enum value. The enum must start at 0 and be contiguous. It must // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; template <typename T> T ConsumeEnum() { @@ -183,10 +187,60 @@ public: 0, static_cast<uint32_t>(T::kMaxValue))); } + // Returns a floating point number in the range [0.0, 1.0]. If there's no + // input data left, always returns 0. + template <typename T> T ConsumeProbability() { + static_assert(std::is_floating_point<T>::value, + "A floating point type is required."); + + // Use different integral types for different floating point types in order + // to provide better density of the resulting values. + using IntegralType = + typename std::conditional<(sizeof(T) <= sizeof(uint32_t)), uint32_t, + uint64_t>::type; + + T result = static_cast<T>(ConsumeIntegral<IntegralType>()); + result /= static_cast<T>(std::numeric_limits<IntegralType>::max()); + return result; + } + + // Returns a floating point value in the range [Type's lowest, Type's max] by + // consuming bytes from the input data. If there's no input data left, always + // returns approximately 0. + template <typename T> T ConsumeFloatingPoint() { + return ConsumeFloatingPointInRange<T>(std::numeric_limits<T>::lowest(), + std::numeric_limits<T>::max()); + } + + // Returns a floating point value in the given range by consuming bytes from + // the input data. If there's no input data left, returns |min|. Note that + // |min| must be less than or equal to |max|. + template <typename T> T ConsumeFloatingPointInRange(T min, T max) { + if (min > max) + abort(); + + T range = .0; + T result = min; + constexpr T zero(.0); + if (max > zero && min < zero && max > min + std::numeric_limits<T>::max()) { + // The diff |max - min| would overflow the given floating point type. Use + // the half of the diff as the range and consume a bool to decide whether + // the result is in the first of the second part of the diff. + range = (max / 2.0) - (min / 2.0); + if (ConsumeBool()) { + result += range; + } + } else { + range = max - min; + } + + return result + range * ConsumeProbability<T>(); + } + // Reports the remaining bytes available for fuzzed input. size_t remaining_bytes() { return remaining_bytes_; } -private: + private: FuzzedDataProvider(const FuzzedDataProvider &) = delete; FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; @@ -209,6 +263,12 @@ private: // which seems to be a natural choice for other implementations as well. // To increase the odds even more, we also call |shrink_to_fit| below. std::vector<T> result(size); + if (size == 0) { + if (num_bytes_to_consume != 0) + abort(); + return result; + } + std::memcpy(result.data(), data_ptr_, num_bytes_to_consume); Advance(num_bytes_to_consume); @@ -230,9 +290,9 @@ private: // Avoid using implementation-defined unsigned to signer conversions. // To learn more, see https://stackoverflow.com/questions/13150449. - if (value <= std::numeric_limits<TS>::max()) + if (value <= std::numeric_limits<TS>::max()) { return static_cast<TS>(value); - else { + } else { constexpr auto TS_min = std::numeric_limits<TS>::min(); return TS_min + static_cast<char>(value - TS_min); } diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp new file mode 100644 index 0000000000..cb0fbdf76f --- /dev/null +++ b/src/test/fuzz/base_encode_decode.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> + +#include <base58.h> +#include <util/string.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_encoded_string(buffer.begin(), buffer.end()); + + std::vector<unsigned char> decoded; + if (DecodeBase58(random_encoded_string, decoded, 100)) { + const std::string encoded_string = EncodeBase58(decoded); + assert(encoded_string == TrimString(encoded_string)); + assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); + } + + if (DecodeBase58Check(random_encoded_string, decoded, 100)) { + const std::string encoded_string = EncodeBase58Check(decoded); + assert(encoded_string == TrimString(encoded_string)); + assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); + } + + bool pf_invalid; + std::string decoded_string = DecodeBase32(random_encoded_string, &pf_invalid); + if (!pf_invalid) { + const std::string encoded_string = EncodeBase32(decoded_string); + assert(encoded_string == TrimString(encoded_string)); + assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); + } + + decoded_string = DecodeBase64(random_encoded_string, &pf_invalid); + if (!pf_invalid) { + const std::string encoded_string = EncodeBase64(decoded_string); + assert(encoded_string == TrimString(encoded_string)); + assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); + } +} diff --git a/src/test/fuzz/bech32.cpp b/src/test/fuzz/bech32.cpp new file mode 100644 index 0000000000..8b91f9bc96 --- /dev/null +++ b/src/test/fuzz/bech32.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bech32.h> +#include <test/fuzz/fuzz.h> +#include <test/util/str.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <utility> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_string(buffer.begin(), buffer.end()); + const std::pair<std::string, std::vector<uint8_t>> r1 = bech32::Decode(random_string); + if (r1.first.empty()) { + assert(r1.second.empty()); + } else { + const std::string& hrp = r1.first; + const std::vector<uint8_t>& data = r1.second; + const std::string reencoded = bech32::Encode(hrp, data); + assert(CaseInsensitiveEqual(random_string, reencoded)); + } + + std::vector<unsigned char> input; + ConvertBits<8, 5, true>([&](unsigned char c) { input.push_back(c); }, buffer.begin(), buffer.end()); + const std::string encoded = bech32::Encode("bc", input); + assert(!encoded.empty()); + + const std::pair<std::string, std::vector<uint8_t>> r2 = bech32::Decode(encoded); + if (r2.first.empty()) { + assert(r2.second.empty()); + } else { + const std::string& hrp = r2.first; + const std::vector<uint8_t>& data = r2.second; + assert(hrp == "bc"); + assert(data == input); + } +} diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp new file mode 100644 index 0000000000..431248de4a --- /dev/null +++ b/src/test/fuzz/block.cpp @@ -0,0 +1,63 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <consensus/merkle.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <core_memusage.h> +#include <pubkey.h> +#include <primitives/block.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <validation.h> +#include <version.h> + +#include <cassert> +#include <string> + +void initialize() +{ + const static auto verify_handle = MakeUnique<ECCVerifyHandle>(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + CBlock block; + try { + int nVersion; + ds >> nVersion; + ds.SetVersion(nVersion); + ds >> block; + } catch (const std::ios_base::failure&) { + return; + } + const Consensus::Params& consensus_params = Params().GetConsensus(); + BlockValidationState validation_state_pow_and_merkle; + const bool valid_incl_pow_and_merkle = CheckBlock(block, validation_state_pow_and_merkle, consensus_params, /* fCheckPOW= */ true, /* fCheckMerkleRoot= */ true); + BlockValidationState validation_state_pow; + const bool valid_incl_pow = CheckBlock(block, validation_state_pow, consensus_params, /* fCheckPOW= */ true, /* fCheckMerkleRoot= */ false); + BlockValidationState validation_state_merkle; + const bool valid_incl_merkle = CheckBlock(block, validation_state_merkle, consensus_params, /* fCheckPOW= */ false, /* fCheckMerkleRoot= */ true); + BlockValidationState validation_state_none; + const bool valid_incl_none = CheckBlock(block, validation_state_none, consensus_params, /* fCheckPOW= */ false, /* fCheckMerkleRoot= */ false); + if (valid_incl_pow_and_merkle) { + assert(valid_incl_pow && valid_incl_merkle && valid_incl_none); + } else if (valid_incl_merkle || valid_incl_pow) { + assert(valid_incl_none); + } + (void)block.GetHash(); + (void)block.ToString(); + (void)BlockMerkleRoot(block); + if (!block.vtx.empty()) { + // TODO: Avoid array index out of bounds error in BlockWitnessMerkleRoot + // when block.vtx.empty(). + (void)BlockWitnessMerkleRoot(block); + } + (void)GetBlockWeight(block); + (void)GetWitnessCommitmentIndex(block); + (void)RecursiveDynamicUsage(block); +} diff --git a/src/test/fuzz/decode_tx.cpp b/src/test/fuzz/decode_tx.cpp new file mode 100644 index 0000000000..09c4ff05df --- /dev/null +++ b/src/test/fuzz/decode_tx.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <primitives/transaction.h> +#include <test/fuzz/fuzz.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string tx_hex = HexStr(std::string{buffer.begin(), buffer.end()}); + CMutableTransaction mtx; + const bool result_none = DecodeHexTx(mtx, tx_hex, false, false); + const bool result_try_witness = DecodeHexTx(mtx, tx_hex, false, true); + const bool result_try_witness_and_maybe_no_witness = DecodeHexTx(mtx, tx_hex, true, true); + const bool result_try_no_witness = DecodeHexTx(mtx, tx_hex, true, false); + assert(!result_none); + if (result_try_witness_and_maybe_no_witness) { + assert(result_try_no_witness || result_try_witness); + } + // if (result_try_no_witness) { // Uncomment when https://github.com/bitcoin/bitcoin/pull/17775 is merged + if (result_try_witness) { // Remove stop-gap when https://github.com/bitcoin/bitcoin/pull/17775 is merged + assert(result_try_witness_and_maybe_no_witness); + } +} diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index c4c25854fd..47d5038c26 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -3,11 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <pubkey.h> #include <script/descriptor.h> #include <test/fuzz/fuzz.h> +#include <util/memory.h> void initialize() { + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); SelectParams(CBaseChainParams::REGTEST); } diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index bcd8691359..f06f339b9d 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -5,18 +5,25 @@ #include <addrdb.h> #include <addrman.h> #include <blockencodings.h> +#include <blockfilter.h> #include <chain.h> #include <coins.h> #include <compressor.h> #include <consensus/merkle.h> +#include <key.h> +#include <merkleblock.h> #include <net.h> #include <primitives/block.h> #include <protocol.h> +#include <psbt.h> #include <pubkey.h> +#include <script/keyorigin.h> #include <streams.h> #include <undo.h> #include <version.h> +#include <exception> +#include <stdexcept> #include <stdint.h> #include <unistd.h> @@ -30,137 +37,186 @@ void initialize() static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); } -void test_one_input(const std::vector<uint8_t>& buffer) +namespace { + +struct invalid_fuzzing_input_exception : public std::exception { +}; + +template <typename T> +CDataStream Serialize(const T& obj) +{ + CDataStream ds(SER_NETWORK, INIT_PROTO_VERSION); + ds << obj; + return ds; +} + +template <typename T> +T Deserialize(CDataStream ds) +{ + T obj; + ds >> obj; + return obj; +} + +template <typename T> +void DeserializeFromFuzzingInput(const std::vector<uint8_t>& buffer, T& obj) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); try { - int nVersion; - ds >> nVersion; - ds.SetVersion(nVersion); - } catch (const std::ios_base::failure& e) { - return; + int version; + ds >> version; + ds.SetVersion(version); + } catch (const std::ios_base::failure&) { + throw invalid_fuzzing_input_exception(); + } + try { + ds >> obj; + } catch (const std::ios_base::failure&) { + throw invalid_fuzzing_input_exception(); } + assert(buffer.empty() || !Serialize(obj).empty()); +} + +template <typename T> +void AssertEqualAfterSerializeDeserialize(const T& obj) +{ + assert(Deserialize<T>(Serialize(obj)) == obj); +} -#if BLOCK_DESERIALIZE - try - { - CBlock block; - ds >> block; - } catch (const std::ios_base::failure& e) {return;} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + try { +#if BLOCK_FILTER_DESERIALIZE + BlockFilter block_filter; + DeserializeFromFuzzingInput(buffer, block_filter); +#elif ADDR_INFO_DESERIALIZE + CAddrInfo addr_info; + DeserializeFromFuzzingInput(buffer, addr_info); +#elif BLOCK_FILE_INFO_DESERIALIZE + CBlockFileInfo block_file_info; + DeserializeFromFuzzingInput(buffer, block_file_info); +#elif BLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE + CBlockHeaderAndShortTxIDs block_header_and_short_txids; + DeserializeFromFuzzingInput(buffer, block_header_and_short_txids); +#elif FEE_RATE_DESERIALIZE + CFeeRate fee_rate; + DeserializeFromFuzzingInput(buffer, fee_rate); + AssertEqualAfterSerializeDeserialize(fee_rate); +#elif MERKLE_BLOCK_DESERIALIZE + CMerkleBlock merkle_block; + DeserializeFromFuzzingInput(buffer, merkle_block); +#elif OUT_POINT_DESERIALIZE + COutPoint out_point; + DeserializeFromFuzzingInput(buffer, out_point); + AssertEqualAfterSerializeDeserialize(out_point); +#elif PARTIAL_MERKLE_TREE_DESERIALIZE + CPartialMerkleTree partial_merkle_tree; + DeserializeFromFuzzingInput(buffer, partial_merkle_tree); +#elif PUB_KEY_DESERIALIZE + CPubKey pub_key; + DeserializeFromFuzzingInput(buffer, pub_key); + // TODO: The following equivalence should hold for CPubKey? Fix. + // AssertEqualAfterSerializeDeserialize(pub_key); +#elif SCRIPT_DESERIALIZE + CScript script; + DeserializeFromFuzzingInput(buffer, script); +#elif SUB_NET_DESERIALIZE + CSubNet sub_net; + DeserializeFromFuzzingInput(buffer, sub_net); + AssertEqualAfterSerializeDeserialize(sub_net); +#elif TX_IN_DESERIALIZE + CTxIn tx_in; + DeserializeFromFuzzingInput(buffer, tx_in); + AssertEqualAfterSerializeDeserialize(tx_in); +#elif FLAT_FILE_POS_DESERIALIZE + FlatFilePos flat_file_pos; + DeserializeFromFuzzingInput(buffer, flat_file_pos); + AssertEqualAfterSerializeDeserialize(flat_file_pos); +#elif KEY_ORIGIN_INFO_DESERIALIZE + KeyOriginInfo key_origin_info; + DeserializeFromFuzzingInput(buffer, key_origin_info); + AssertEqualAfterSerializeDeserialize(key_origin_info); +#elif PARTIALLY_SIGNED_TRANSACTION_DESERIALIZE + PartiallySignedTransaction partially_signed_transaction; + DeserializeFromFuzzingInput(buffer, partially_signed_transaction); +#elif PREFILLED_TRANSACTION_DESERIALIZE + PrefilledTransaction prefilled_transaction; + DeserializeFromFuzzingInput(buffer, prefilled_transaction); +#elif PSBT_INPUT_DESERIALIZE + PSBTInput psbt_input; + DeserializeFromFuzzingInput(buffer, psbt_input); +#elif PSBT_OUTPUT_DESERIALIZE + PSBTOutput psbt_output; + DeserializeFromFuzzingInput(buffer, psbt_output); +#elif BLOCK_DESERIALIZE + CBlock block; + DeserializeFromFuzzingInput(buffer, block); #elif BLOCKLOCATOR_DESERIALIZE - try - { - CBlockLocator bl; - ds >> bl; - } catch (const std::ios_base::failure& e) {return;} + CBlockLocator bl; + DeserializeFromFuzzingInput(buffer, bl); #elif BLOCKMERKLEROOT - try - { - CBlock block; - ds >> block; - bool mutated; - BlockMerkleRoot(block, &mutated); - } catch (const std::ios_base::failure& e) {return;} + CBlock block; + DeserializeFromFuzzingInput(buffer, block); + bool mutated; + BlockMerkleRoot(block, &mutated); #elif ADDRMAN_DESERIALIZE - try - { - CAddrMan am; - ds >> am; - } catch (const std::ios_base::failure& e) {return;} + CAddrMan am; + DeserializeFromFuzzingInput(buffer, am); #elif BLOCKHEADER_DESERIALIZE - try - { - CBlockHeader bh; - ds >> bh; - } catch (const std::ios_base::failure& e) {return;} + CBlockHeader bh; + DeserializeFromFuzzingInput(buffer, bh); #elif BANENTRY_DESERIALIZE - try - { - CBanEntry be; - ds >> be; - } catch (const std::ios_base::failure& e) {return;} + CBanEntry be; + DeserializeFromFuzzingInput(buffer, be); #elif TXUNDO_DESERIALIZE - try - { - CTxUndo tu; - ds >> tu; - } catch (const std::ios_base::failure& e) {return;} + CTxUndo tu; + DeserializeFromFuzzingInput(buffer, tu); #elif BLOCKUNDO_DESERIALIZE - try - { - CBlockUndo bu; - ds >> bu; - } catch (const std::ios_base::failure& e) {return;} + CBlockUndo bu; + DeserializeFromFuzzingInput(buffer, bu); #elif COINS_DESERIALIZE - try - { - Coin coin; - ds >> coin; - } catch (const std::ios_base::failure& e) {return;} + Coin coin; + DeserializeFromFuzzingInput(buffer, coin); #elif NETADDR_DESERIALIZE - try - { - CNetAddr na; - ds >> na; - } catch (const std::ios_base::failure& e) {return;} + CNetAddr na; + DeserializeFromFuzzingInput(buffer, na); + AssertEqualAfterSerializeDeserialize(na); #elif SERVICE_DESERIALIZE - try - { - CService s; - ds >> s; - } catch (const std::ios_base::failure& e) {return;} + CService s; + DeserializeFromFuzzingInput(buffer, s); + AssertEqualAfterSerializeDeserialize(s); #elif MESSAGEHEADER_DESERIALIZE - CMessageHeader::MessageStartChars pchMessageStart = {0x00, 0x00, 0x00, 0x00}; - try - { - CMessageHeader mh(pchMessageStart); - ds >> mh; - if (!mh.IsValid(pchMessageStart)) {return;} - } catch (const std::ios_base::failure& e) {return;} + const CMessageHeader::MessageStartChars pchMessageStart = {0x00, 0x00, 0x00, 0x00}; + CMessageHeader mh(pchMessageStart); + DeserializeFromFuzzingInput(buffer, mh); + (void)mh.IsValid(pchMessageStart); #elif ADDRESS_DESERIALIZE - try - { - CAddress a; - ds >> a; - } catch (const std::ios_base::failure& e) {return;} + CAddress a; + DeserializeFromFuzzingInput(buffer, a); #elif INV_DESERIALIZE - try - { - CInv i; - ds >> i; - } catch (const std::ios_base::failure& e) {return;} + CInv i; + DeserializeFromFuzzingInput(buffer, i); #elif BLOOMFILTER_DESERIALIZE - try - { - CBloomFilter bf; - ds >> bf; - } catch (const std::ios_base::failure& e) {return;} + CBloomFilter bf; + DeserializeFromFuzzingInput(buffer, bf); #elif DISKBLOCKINDEX_DESERIALIZE - try - { - CDiskBlockIndex dbi; - ds >> dbi; - } catch (const std::ios_base::failure& e) {return;} + CDiskBlockIndex dbi; + DeserializeFromFuzzingInput(buffer, dbi); #elif TXOUTCOMPRESSOR_DESERIALIZE - CTxOut to; - CTxOutCompressor toc(to); - try - { - ds >> toc; - } catch (const std::ios_base::failure& e) {return;} + CTxOut to; + auto toc = Using<TxOutCompression>(to); + DeserializeFromFuzzingInput(buffer, toc); #elif BLOCKTRANSACTIONS_DESERIALIZE - try - { - BlockTransactions bt; - ds >> bt; - } catch (const std::ios_base::failure& e) {return;} + BlockTransactions bt; + DeserializeFromFuzzingInput(buffer, bt); #elif BLOCKTRANSACTIONSREQUEST_DESERIALIZE - try - { - BlockTransactionsRequest btr; - ds >> btr; - } catch (const std::ios_base::failure& e) {return;} + BlockTransactionsRequest btr; + DeserializeFromFuzzingInput(buffer, btr); #else #error Need at least one fuzz target to compile #endif + } catch (const invalid_fuzzing_input_exception&) { + } } diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index 9444cd489e..7acdd76857 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -2,12 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <pubkey.h> #include <script/interpreter.h> -#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <util/memory.h> #include <limits> +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index da4e623e98..a085e36911 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -4,10 +4,15 @@ #include <test/fuzz/fuzz.h> +#include <test/util/setup_common.h> + #include <cstdint> #include <unistd.h> #include <vector> +const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; + +#if defined(__AFL_COMPILER) static bool read_stdin(std::vector<uint8_t>& data) { uint8_t buffer[1024]; @@ -19,6 +24,7 @@ static bool read_stdin(std::vector<uint8_t>& data) } return length == 0; } +#endif // Default initialization: Override using a non-weak initialize(). __attribute__((weak)) void initialize() @@ -40,9 +46,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -// Declare main(...) "weak" to allow for libFuzzer linking. libFuzzer provides -// the main(...) function. -__attribute__((weak)) int main(int argc, char** argv) +// Generally, the fuzzer will provide main(), except for AFL +#if defined(__AFL_COMPILER) +int main(int argc, char** argv) { initialize(); #ifdef __AFL_INIT @@ -70,3 +76,4 @@ __attribute__((weak)) int main(int argc, char** argv) #endif return 0; } +#endif diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp new file mode 100644 index 0000000000..54693180be --- /dev/null +++ b/src/test/fuzz/hex.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> + +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_hex_string(buffer.begin(), buffer.end()); + const std::vector<unsigned char> data = ParseHex(random_hex_string); + const std::string hex_data = HexStr(data); + if (IsHex(random_hex_string)) { + assert(ToLower(random_hex_string) == hex_data); + } +} diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp new file mode 100644 index 0000000000..723938bcdb --- /dev/null +++ b/src/test/fuzz/integer.cpp @@ -0,0 +1,127 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <arith_uint256.h> +#include <compressor.h> +#include <consensus/merkle.h> +#include <core_io.h> +#include <crypto/common.h> +#include <crypto/siphash.h> +#include <key_io.h> +#include <memusage.h> +#include <netbase.h> +#include <policy/settings.h> +#include <pow.h> +#include <pubkey.h> +#include <rpc/util.h> +#include <script/signingprovider.h> +#include <script/standard.h> +#include <serialize.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <uint256.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <util/time.h> + +#include <cassert> +#include <limits> +#include <vector> + +void initialize() +{ + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + if (buffer.size() < sizeof(uint256) + sizeof(uint160)) { + return; + } + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const uint256 u256(fuzzed_data_provider.ConsumeBytes<unsigned char>(sizeof(uint256))); + const uint160 u160(fuzzed_data_provider.ConsumeBytes<unsigned char>(sizeof(uint160))); + const uint64_t u64 = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); + const int64_t i64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const uint32_t u32 = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + const int32_t i32 = fuzzed_data_provider.ConsumeIntegral<int32_t>(); + const uint16_t u16 = fuzzed_data_provider.ConsumeIntegral<uint16_t>(); + const int16_t i16 = fuzzed_data_provider.ConsumeIntegral<int16_t>(); + const uint8_t u8 = fuzzed_data_provider.ConsumeIntegral<uint8_t>(); + const int8_t i8 = fuzzed_data_provider.ConsumeIntegral<int8_t>(); + // We cannot assume a specific value of std::is_signed<char>::value: + // ConsumeIntegral<char>() instead of casting from {u,}int8_t. + const char ch = fuzzed_data_provider.ConsumeIntegral<char>(); + + const Consensus::Params& consensus_params = Params().GetConsensus(); + (void)CheckProofOfWork(u256, u32, consensus_params); + (void)CompressAmount(u64); + static const uint256 u256_min(uint256S("0000000000000000000000000000000000000000000000000000000000000000")); + static const uint256 u256_max(uint256S("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + const std::vector<uint256> v256{u256, u256_min, u256_max}; + (void)ComputeMerkleRoot(v256); + (void)CountBits(u64); + (void)DecompressAmount(u64); + (void)FormatISO8601Date(i64); + (void)FormatISO8601DateTime(i64); + (void)GetSizeOfCompactSize(u64); + (void)GetSpecialScriptSize(u32); + // (void)GetVirtualTransactionSize(i64, i64); // function defined only for a subset of int64_t inputs + // (void)GetVirtualTransactionSize(i64, i64, u32); // function defined only for a subset of int64_t/uint32_t inputs + (void)HexDigit(ch); + (void)i64tostr(i64); + (void)IsDigit(ch); + (void)IsSpace(ch); + (void)IsSwitchChar(ch); + (void)itostr(i32); + (void)memusage::DynamicUsage(ch); + (void)memusage::DynamicUsage(i16); + (void)memusage::DynamicUsage(i32); + (void)memusage::DynamicUsage(i64); + (void)memusage::DynamicUsage(i8); + (void)memusage::DynamicUsage(u16); + (void)memusage::DynamicUsage(u32); + (void)memusage::DynamicUsage(u64); + (void)memusage::DynamicUsage(u8); + const unsigned char uch = static_cast<unsigned char>(u8); + (void)memusage::DynamicUsage(uch); + (void)MillisToTimeval(i64); + const double d = ser_uint64_to_double(u64); + assert(ser_double_to_uint64(d) == u64); + const float f = ser_uint32_to_float(u32); + assert(ser_float_to_uint32(f) == u32); + (void)SighashToStr(uch); + (void)SipHashUint256(u64, u64, u256); + (void)SipHashUint256Extra(u64, u64, u256, u32); + (void)ToLower(ch); + + const arith_uint256 au256 = UintToArith256(u256); + assert(ArithToUint256(au256) == u256); + assert(uint256S(au256.GetHex()) == u256); + (void)au256.bits(); + (void)au256.GetCompact(/* fNegative= */ false); + (void)au256.GetCompact(/* fNegative= */ true); + (void)au256.getdouble(); + (void)au256.GetHex(); + (void)au256.GetLow64(); + (void)au256.size(); + (void)au256.ToString(); + + const CKeyID key_id{u160}; + const CScriptID script_id{u160}; + // CTxDestination = CNoDestination ∪ PKHash ∪ ScriptHash ∪ WitnessV0ScriptHash ∪ WitnessV0KeyHash ∪ WitnessUnknown + const PKHash pk_hash{u160}; + const ScriptHash script_hash{u160}; + const WitnessV0KeyHash witness_v0_key_hash{u160}; + const WitnessV0ScriptHash witness_v0_script_hash{u256}; + const std::vector<CTxDestination> destinations{pk_hash, script_hash, witness_v0_key_hash, witness_v0_script_hash}; + const SigningProvider store; + for (const CTxDestination& destination : destinations) { + (void)DescribeAddress(destination); + (void)EncodeDestination(destination); + (void)GetKeyForDestination(store, destination); + (void)GetScriptForDestination(destination); + (void)IsValidDestination(destination); + } +} diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp new file mode 100644 index 0000000000..9a23f4b2d4 --- /dev/null +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> +#include <util/bip32.h> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string keypath_str(buffer.begin(), buffer.end()); + std::vector<uint32_t> keypath; + (void)ParseHDKeypath(keypath_str, keypath); +} diff --git a/src/test/fuzz/parse_iso8601.cpp b/src/test/fuzz/parse_iso8601.cpp new file mode 100644 index 0000000000..c86f8a853e --- /dev/null +++ b/src/test/fuzz/parse_iso8601.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <util/time.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const int64_t random_time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const std::string random_string = fuzzed_data_provider.ConsumeRemainingBytesAsString(); + + const std::string iso8601_datetime = FormatISO8601DateTime(random_time); + const int64_t parsed_time_1 = ParseISO8601DateTime(iso8601_datetime); + if (random_time >= 0) { + assert(parsed_time_1 >= 0); + if (iso8601_datetime.length() == 20) { + assert(parsed_time_1 == random_time); + } + } + + const int64_t parsed_time_2 = ParseISO8601DateTime(random_string); + assert(parsed_time_2 >= 0); +} diff --git a/src/test/fuzz/parse_numbers.cpp b/src/test/fuzz/parse_numbers.cpp new file mode 100644 index 0000000000..59f89dc9fb --- /dev/null +++ b/src/test/fuzz/parse_numbers.cpp @@ -0,0 +1,35 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> +#include <util/moneystr.h> +#include <util/strencodings.h> + +#include <string> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_string(buffer.begin(), buffer.end()); + + CAmount amount; + (void)ParseMoney(random_string, amount); + + double d; + (void)ParseDouble(random_string, &d); + + int32_t i32; + (void)ParseInt32(random_string, &i32); + (void)atoi(random_string); + + uint32_t u32; + (void)ParseUInt32(random_string, &u32); + + int64_t i64; + (void)atoi64(random_string); + (void)ParseFixedPoint(random_string, 3, &i64); + (void)ParseInt64(random_string, &i64); + + uint64_t u64; + (void)ParseUInt64(random_string, &u64); +} diff --git a/src/test/fuzz/parse_script.cpp b/src/test/fuzz/parse_script.cpp new file mode 100644 index 0000000000..21ac1aecf3 --- /dev/null +++ b/src/test/fuzz/parse_script.cpp @@ -0,0 +1,16 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <script/script.h> +#include <test/fuzz/fuzz.h> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string script_string(buffer.begin(), buffer.end()); + try { + (void)ParseScript(script_string); + } catch (const std::runtime_error&) { + } +} diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp new file mode 100644 index 0000000000..3ad112dbad --- /dev/null +++ b/src/test/fuzz/parse_univalue.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <core_io.h> +#include <rpc/client.h> +#include <rpc/util.h> +#include <test/fuzz/fuzz.h> +#include <util/memory.h> + +#include <limits> +#include <string> + +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_string(buffer.begin(), buffer.end()); + bool valid = true; + const UniValue univalue = [&] { + try { + return ParseNonRFCJSONValue(random_string); + } catch (const std::runtime_error&) { + valid = false; + return NullUniValue; + } + }(); + if (!valid) { + return; + } + try { + (void)ParseHashO(univalue, "A"); + (void)ParseHashO(univalue, random_string); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseHashV(univalue, "A"); + (void)ParseHashV(univalue, random_string); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseHexO(univalue, "A"); + (void)ParseHexO(univalue, random_string); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseHexUV(univalue, "A"); + (void)ParseHexUV(univalue, random_string); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseHexV(univalue, "A"); + (void)ParseHexV(univalue, random_string); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseSighashString(univalue); + } catch (const std::runtime_error&) { + } + try { + (void)AmountFromValue(univalue); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + FlatSigningProvider provider; + (void)EvalDescriptorStringOrObject(univalue, provider); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseConfirmTarget(univalue, std::numeric_limits<unsigned int>::max()); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } + try { + (void)ParseDescriptorRange(univalue); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } +} diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp new file mode 100644 index 0000000000..1ce28f9a6d --- /dev/null +++ b/src/test/fuzz/psbt.cpp @@ -0,0 +1,79 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/fuzz.h> + +#include <node/psbt.h> +#include <optional.h> +#include <psbt.h> +#include <pubkey.h> +#include <script/script.h> +#include <streams.h> +#include <util/memory.h> +#include <version.h> + +#include <cstdint> +#include <string> +#include <vector> + +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + PartiallySignedTransaction psbt_mut; + const std::string raw_psbt{buffer.begin(), buffer.end()}; + std::string error; + if (!DecodeRawPSBT(psbt_mut, raw_psbt, error)) { + return; + } + const PartiallySignedTransaction psbt = psbt_mut; + + const PSBTAnalysis analysis = AnalyzePSBT(psbt); + (void)PSBTRoleName(analysis.next); + for (const PSBTInputAnalysis& input_analysis : analysis.inputs) { + (void)PSBTRoleName(input_analysis.next); + } + + (void)psbt.IsNull(); + (void)psbt.IsSane(); + + Optional<CMutableTransaction> tx = psbt.tx; + if (tx) { + const CMutableTransaction& mtx = *tx; + const PartiallySignedTransaction psbt_from_tx{mtx}; + } + + for (const PSBTInput& input : psbt.inputs) { + (void)PSBTInputSigned(input); + (void)input.IsNull(); + (void)input.IsSane(); + } + + for (const PSBTOutput& output : psbt.outputs) { + (void)output.IsNull(); + } + + for (size_t i = 0; i < psbt.tx->vin.size(); ++i) { + CTxOut tx_out; + if (psbt.GetInputUTXO(tx_out, i)) { + (void)tx_out.IsNull(); + (void)tx_out.ToString(); + } + } + + psbt_mut = psbt; + (void)FinalizePSBT(psbt_mut); + + psbt_mut = psbt; + CMutableTransaction result; + if (FinalizeAndExtractPSBT(psbt_mut, result)) { + const PartiallySignedTransaction psbt_from_tx{result}; + } + + psbt_mut = psbt; + (void)psbt_mut.Merge(psbt); +} diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 0bf5cd5c72..08622d0979 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -2,8 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <pubkey.h> #include <script/interpreter.h> #include <streams.h> +#include <util/memory.h> #include <version.h> #include <test/fuzz/fuzz.h> @@ -11,6 +13,11 @@ /** Flags that are not forbidden by an assert */ static bool IsValidFlagCombination(unsigned flags); +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + void test_one_input(const std::vector<uint8_t>& buffer) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp new file mode 100644 index 0000000000..0de21f0e7c --- /dev/null +++ b/src/test/fuzz/strprintf.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <tinyformat.h> +#include <util/strencodings.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::string format_string = fuzzed_data_provider.ConsumeRandomLengthString(64); + + const int digits_in_format_specifier = std::count_if(format_string.begin(), format_string.end(), IsDigit); + + // Avoid triggering the following crash bug: + // * strprintf("%987654321000000:", 1); + // + // Avoid triggering the following OOM bug: + // * strprintf("%.222222200000000$", 1.1); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + if (format_string.find("%") != std::string::npos && digits_in_format_specifier >= 7) { + return; + } + + // Avoid triggering the following crash bug: + // * strprintf("%1$*1$*", -11111111); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + if (format_string.find("%") != std::string::npos && format_string.find("$") != std::string::npos && format_string.find("*") != std::string::npos && digits_in_format_specifier > 0) { + return; + } + + // Avoid triggering the following crash bug: + // * strprintf("%.1s", (char*)nullptr); + // + // (void)strprintf(format_string, (char*)nullptr); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + + try { + (void)strprintf(format_string, (signed char*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (unsigned char*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (void*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (bool*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (float*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (double*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int16_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint16_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int32_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint32_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int64_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint64_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + + try { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 13)) { + case 0: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + break; + case 1: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + break; + case 2: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + break; + case 3: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + break; + case 4: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); + break; + case 5: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); + break; + case 6: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + break; + case 7: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + break; + case 8: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + break; + case 9: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + break; + case 10: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + break; + case 11: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + break; + case 12: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + break; + case 13: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + default: + assert(false); + } + } catch (const tinyformat::format_error&) { + } +} diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 383d879040..fefafda36b 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -26,23 +26,35 @@ void test_one_input(const std::vector<uint8_t>& buffer) int nVersion; ds >> nVersion; ds.SetVersion(nVersion); - } catch (const std::ios_base::failure& e) { + } catch (const std::ios_base::failure&) { return; } - bool valid = true; + bool valid_tx = true; const CTransaction tx = [&] { try { return CTransaction(deserialize, ds); - } catch (const std::ios_base::failure& e) { - valid = false; + } catch (const std::ios_base::failure&) { + valid_tx = false; return CTransaction(); } }(); - if (!valid) { + bool valid_mutable_tx = true; + CDataStream ds_mtx(buffer, SER_NETWORK, INIT_PROTO_VERSION); + CMutableTransaction mutable_tx; + try { + int nVersion; + ds_mtx >> nVersion; + ds_mtx.SetVersion(nVersion); + ds_mtx >> mutable_tx; + } catch (const std::ios_base::failure&) { + valid_mutable_tx = false; + } + assert(valid_tx == valid_mutable_tx); + if (!valid_tx) { return; } - CValidationState state_with_dupe_check; + TxValidationState state_with_dupe_check; (void)CheckTransaction(tx, state_with_dupe_check); const CFeeRate dust_relay_fee{DUST_RELAY_TX_FEE}; diff --git a/src/test/fuzz/tx_in.cpp b/src/test/fuzz/tx_in.cpp new file mode 100644 index 0000000000..8e116537d1 --- /dev/null +++ b/src/test/fuzz/tx_in.cpp @@ -0,0 +1,33 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <core_memusage.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <version.h> + +#include <cassert> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + CTxIn tx_in; + try { + int version; + ds >> version; + ds.SetVersion(version); + ds >> tx_in; + } catch (const std::ios_base::failure&) { + return; + } + + (void)GetTransactionInputWeight(tx_in); + (void)GetVirtualTransactionInputSize(tx_in); + (void)RecursiveDynamicUsage(tx_in); + + (void)tx_in.ToString(); +} diff --git a/src/test/fuzz/tx_out.cpp b/src/test/fuzz/tx_out.cpp new file mode 100644 index 0000000000..aa1338d5ba --- /dev/null +++ b/src/test/fuzz/tx_out.cpp @@ -0,0 +1,35 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <core_memusage.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <version.h> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + CTxOut tx_out; + try { + int version; + ds >> version; + ds.SetVersion(version); + ds >> tx_out; + } catch (const std::ios_base::failure&) { + return; + } + + const CFeeRate dust_relay_fee{DUST_RELAY_TX_FEE}; + (void)GetDustThreshold(tx_out, dust_relay_fee); + (void)IsDust(tx_out, dust_relay_fee); + (void)RecursiveDynamicUsage(tx_out); + + (void)tx_out.ToString(); + (void)tx_out.IsNull(); + tx_out.SetNull(); + assert(tx_out.IsNull()); +} diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 77304fe918..10fb05ca8a 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -4,7 +4,7 @@ #include <util/strencodings.h> #include <util/system.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <utility> @@ -43,7 +43,7 @@ static void SetupArgs(const std::vector<std::pair<std::string, unsigned int>>& a BOOST_AUTO_TEST_CASE(boolarg) { - const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_BOOL); + const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); SetupArgs({foo}); ResetArgs("-foo"); BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); @@ -97,8 +97,8 @@ BOOST_AUTO_TEST_CASE(boolarg) BOOST_AUTO_TEST_CASE(stringarg) { - const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_STRING); - const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_STRING); + const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); + const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs(""); BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", ""), ""); @@ -124,8 +124,8 @@ BOOST_AUTO_TEST_CASE(stringarg) BOOST_AUTO_TEST_CASE(intarg) { - const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_INT); - const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_INT); + const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); + const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs(""); BOOST_CHECK_EQUAL(gArgs.GetArg("-foo", 11), 11); @@ -159,8 +159,8 @@ BOOST_AUTO_TEST_CASE(doubledash) BOOST_AUTO_TEST_CASE(boolargno) { - const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_BOOL); - const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_BOOL); + const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); + const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs("-nofoo"); BOOST_CHECK(!gArgs.GetBoolArg("-foo", true)); @@ -183,4 +183,32 @@ BOOST_AUTO_TEST_CASE(boolargno) BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); } +BOOST_AUTO_TEST_CASE(logargs) +{ + const auto okaylog_bool = std::make_pair("-okaylog-bool", ArgsManager::ALLOW_BOOL); + const auto okaylog_negbool = std::make_pair("-okaylog-negbool", ArgsManager::ALLOW_BOOL); + const auto okaylog = std::make_pair("-okaylog", ArgsManager::ALLOW_ANY); + const auto dontlog = std::make_pair("-dontlog", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE); + SetupArgs({okaylog_bool, okaylog_negbool, okaylog, dontlog}); + ResetArgs("-okaylog-bool -nookaylog-negbool -okaylog=public -dontlog=private"); + + // Everything logged to debug.log will also append to str + std::string str; + auto print_connection = LogInstance().PushBackCallback( + [&str](const std::string& s) { + str += s; + }); + + // Log the arguments + gArgs.LogArgs(); + + LogInstance().DeleteCallback(print_connection); + // Check that what should appear does, and what shouldn't doesn't. + BOOST_CHECK(str.find("Command-line arg: okaylog-bool=\"\"") != std::string::npos); + BOOST_CHECK(str.find("Command-line arg: okaylog-negbool=false") != std::string::npos); + BOOST_CHECK(str.find("Command-line arg: okaylog=\"public\"") != std::string::npos); + BOOST_CHECK(str.find("dontlog=****") != std::string::npos); + BOOST_CHECK(str.find("private") == std::string::npos); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d91fcb0034..b864e6e599 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -6,7 +6,7 @@ #include <crypto/siphash.h> #include <hash.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index e924f27d1b..b52513f4af 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -9,7 +9,7 @@ #include <key_io.h> #include <script/script.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_properties.cpp b/src/test/key_properties.cpp index 95587130fc..0e45a2549d 100644 --- a/src/test/key_properties.cpp +++ b/src/test/key_properties.cpp @@ -4,7 +4,7 @@ #include <key.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 3e99dcaa40..85dc961bea 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -8,7 +8,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <vector> diff --git a/src/test/limitedmap_tests.cpp b/src/test/limitedmap_tests.cpp index 00b36f51fb..ea18debbd3 100644 --- a/src/test/limitedmap_tests.cpp +++ b/src/test/limitedmap_tests.cpp @@ -4,7 +4,7 @@ #include <limitedmap.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp new file mode 100644 index 0000000000..25655b8894 --- /dev/null +++ b/src/test/logging_tests.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <logging/timer.h> +#include <test/util/setup_common.h> + +#include <chrono> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(logging_timer) +{ + + SetMockTime(1); + auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); + + SetMockTime(1); + auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)"); + + SetMockTime(1); + auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000.00μs)"); + + SetMockTime(0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/main.cpp b/src/test/main.cpp index ff3f36b561..e6529949e2 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -2,6 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +/** + * See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/link_references/link_boost_test_module_macro.html + */ #define BOOST_TEST_MODULE Bitcoin Core Test Suite #include <boost/test/unit_test.hpp> + +#include <test/util/setup_common.h> + +/** Redirect debug log to boost log */ +const std::function<void(const std::string&)> G_TEST_LOG_FUN = [](const std::string& s) { + if (s.back() == '\n') { + // boost will insert the new line + BOOST_TEST_MESSAGE(s.substr(0, s.size() - 1)); + } else { + BOOST_TEST_MESSAGE(s); + } +}; diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index fe5d31b7d3..38fed51af2 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -7,7 +7,7 @@ #include <util/system.h> #include <util/time.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <vector> @@ -749,6 +749,43 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) pool.GetTransactionAncestry(ty6->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, 9ULL); BOOST_CHECK_EQUAL(descendants, 6ULL); + + /* Ancestors represented more than once ("diamond") */ + // + // [ta].0 <- [tb].0 -----<------- [td].0 + // | | + // \---1 <- [tc].0 --<--/ + // + CTransactionRef ta, tb, tc, td; + ta = make_tx(/* output_values */ {10 * COIN}); + tb = make_tx(/* output_values */ {5 * COIN, 3 * COIN}, /* inputs */ {ta}); + tc = make_tx(/* output_values */ {2 * COIN}, /* inputs */ {tb}, /* input_indices */ {1}); + td = make_tx(/* output_values */ {6 * COIN}, /* inputs */ {tb, tc}, /* input_indices */ {0, 0}); + pool.clear(); + pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tc)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(td)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =================== =========== + // ta 1 (ta 4 (ta,tb,tc,td) + // tb 2 (ta,tb) 4 (ta,tb,tc,td) + // tc 3 (ta,tb,tc) 4 (ta,tb,tc,td) + // td 4 (ta,tb,tc,td) 4 (ta,tb,tc,td) + pool.GetTransactionAncestry(ta->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tb->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tc->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(td->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 4ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index dc38a1a818..03dce552fc 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/merkle.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -345,8 +345,8 @@ BOOST_AUTO_TEST_CASE(merkle_test_BlockWitness) hashes[0].SetNull(); hashes[1] = block.vtx[1]->GetHash(); - uint256 merkelRootofHashes = ComputeMerkleRoot(hashes); + uint256 merkleRootofHashes = ComputeMerkleRoot(hashes); - BOOST_CHECK_EQUAL(merkelRootofHashes, blockWitness); + BOOST_CHECK_EQUAL(merkleRootofHashes, blockWitness); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index eac43471c7..9f8c4ba5c5 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -4,7 +4,7 @@ #include <merkleblock.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index c9661b730d..9f3ca87206 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -17,13 +17,24 @@ #include <util/time.h> #include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <memory> #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(miner_tests, TestingSetup) +namespace miner_tests { +struct MinerTestingSetup : public TestingSetup { + void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); + bool TestSequenceLocks(const CTransaction& tx, int flags) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) + { + return CheckSequenceLocks(*m_node.mempool, tx, flags); + } + BlockAssembler AssemblerForTest(const CChainParams& params); +}; +} // namespace miner_tests + +BOOST_FIXTURE_TEST_SUITE(miner_tests, MinerTestingSetup) // BOOST_CHECK_EXCEPTION predicates to check the specific validation error class HasReason { @@ -38,16 +49,16 @@ private: static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); -static BlockAssembler AssemblerForTest(const CChainParams& params) { +BlockAssembler MinerTestingSetup::AssemblerForTest(const CChainParams& params) +{ BlockAssembler::Options options; options.nBlockMaxWeight = MAX_BLOCK_WEIGHT; options.blockMinFeeRate = blockMinFeeRate; - return BlockAssembler(params, options); + return BlockAssembler(*m_node.mempool, params, options); } -static -struct { +constexpr static struct { unsigned char extranonce; unsigned int nonce; } blockinfo[] = { @@ -89,16 +100,10 @@ static CBlockIndex CreateBlockIndex(int nHeight) EXCLUSIVE_LOCKS_REQUIRED(cs_mai return index; } -static bool TestSequenceLocks(const CTransaction &tx, int flags) EXCLUSIVE_LOCKS_REQUIRED(cs_main) -{ - LOCK(::mempool.cs); - return CheckSequenceLocks(::mempool, tx, flags); -} - // Test suite for ancestor feerate transaction selection. // Implemented as an additional function, rather than a separate test case, // to allow reusing the blockchain created in CreateNewBlock_validity. -static void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs) +void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) { // Test the ancestor feerate transaction selection. TestMemPoolEntryHelper entry; @@ -114,19 +119,19 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis uint256 hashParentTx = tx.GetHash(); // save this txid for later use - mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; uint256 hashMediumFeeTx = tx.GetHash(); - mempool.addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); // This tx has a high fee, but depends on the first transaction tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee uint256 hashHighFeeTx = tx.GetHash(); - mempool.addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); @@ -137,7 +142,7 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& tx.vin[0].prevout.hash = hashHighFeeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee uint256 hashFreeTx = tx.GetHash(); - mempool.addUnchecked(entry.Fee(0).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(0).FromTx(tx)); size_t freeTxSize = ::GetSerializeSize(tx, PROTOCOL_VERSION); // Calculate a fee on child transaction that will put the package just @@ -147,7 +152,7 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& tx.vin[0].prevout.hash = hashFreeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse; uint256 hashLowFeeTx = tx.GetHash(); - mempool.addUnchecked(entry.Fee(feeToUse).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(feeToUse).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); // Verify that the free tx and the low fee tx didn't get selected for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { @@ -158,10 +163,10 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& // Test that packages above the min relay fee do get included, even if one // of the transactions is below the min relay fee // Remove the low fee transaction and replace with a higher fee transaction - mempool.removeRecursive(CTransaction(tx), MemPoolRemovalReason::REPLACED); + m_node.mempool->removeRecursive(CTransaction(tx), MemPoolRemovalReason::REPLACED); tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); - mempool.addUnchecked(entry.Fee(feeToUse+2).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(feeToUse+2).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); @@ -174,7 +179,7 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& tx.vout[0].nValue = 5000000000LL - 100000000; tx.vout[1].nValue = 100000000; // 1BTC output uint256 hashFreeTx2 = tx.GetHash(); - mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); // This tx can't be mined by itself tx.vin[0].prevout.hash = hashFreeTx2; @@ -182,7 +187,7 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& feeToUse = blockMinFeeRate.GetFee(freeTxSize); tx.vout[0].nValue = 5000000000LL - 100000000 - feeToUse; uint256 hashLowFeeTx2 = tx.GetHash(); - mempool.addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); // Verify that this tx isn't selected. @@ -195,7 +200,7 @@ static void TestPackageSelection(const CChainParams& chainparams, const CScript& // as well. tx.vin[0].prevout.n = 1; tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee - mempool.addUnchecked(entry.Fee(10000).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(10000).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } @@ -221,7 +226,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // We can't make transactions until we have inputs - // Therefore, load 100 blocks :) + // Therefore, load 110 blocks :) + static_assert(sizeof(blockinfo) / sizeof(*blockinfo) == 110, "Should have 110 blocks to import"); int baseheight = 0; std::vector<CTransactionRef> txFirst; for (unsigned int i = 0; i < sizeof(blockinfo)/sizeof(*blockinfo); ++i) @@ -252,7 +258,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) } LOCK(cs_main); - LOCK(::mempool.cs); + LOCK(m_node.mempool->cs); // Just to make sure we can still make simple blocks BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); @@ -276,12 +282,12 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase // If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); - mempool.clear(); + m_node.mempool->clear(); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); tx.vout[0].nValue = BLOCKSUBSIDY; @@ -291,11 +297,11 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase // If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); tx.vin[0].prevout.hash = hash; } BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - mempool.clear(); + m_node.mempool->clear(); // block size > limit tx.vin[0].scriptSig = CScript(); @@ -311,24 +317,24 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - mempool.clear(); + m_node.mempool->clear(); - // orphan in mempool, template creation fails + // orphan in *m_node.mempool, template creation fails hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); - mempool.clear(); + m_node.mempool->clear(); // child with higher feerate than parent tx.vin[0].scriptSig = CScript() << OP_1; tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin.resize(2); tx.vin[1].scriptSig = CScript() << OP_1; @@ -336,34 +342,34 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[1].prevout.n = 0; tx.vout[0].nValue = tx.vout[0].nValue+BLOCKSUBSIDY-HIGHERFEE; //First txn output + fresh coinbase - new txn fee hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - mempool.clear(); + m_node.mempool->clear(); - // coinbase in mempool, template creation fails + // coinbase in *m_node.mempool, template creation fails tx.vin.resize(1); tx.vin[0].prevout.SetNull(); tx.vin[0].scriptSig = CScript() << OP_0 << OP_1; tx.vout[0].nValue = 0; hash = tx.GetHash(); // give it a fee so it'll get mined - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); // Should throw bad-cb-multiple BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); - mempool.clear(); + m_node.mempool->clear(); - // double spend txn pair in mempool, template creation fails + // double spend txn pair in *m_node.mempool, template creation fails tx.vin[0].prevout.hash = txFirst[0]->GetHash(); tx.vin[0].scriptSig = CScript() << OP_1; tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; tx.vout[0].scriptPubKey = CScript() << OP_1; hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); - mempool.clear(); + m_node.mempool->clear(); // subsidy changing int nHeight = ::ChainActive().Height(); @@ -392,7 +398,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) } BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - // invalid p2sh txn in mempool, template creation fails + // invalid p2sh txn in *m_node.mempool, template creation fails tx.vin[0].prevout.hash = txFirst[0]->GetHash(); tx.vin[0].prevout.n = 0; tx.vin[0].scriptSig = CScript() << OP_1; @@ -400,15 +406,15 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) script = CScript() << OP_0; tx.vout[0].scriptPubKey = GetScriptForDestination(ScriptHash(script)); hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); // Should throw block-validation-failed BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); - mempool.clear(); + m_node.mempool->clear(); // Delete the dummy blocks again. while (::ChainActive().Tip()->nHeight > nHeight) { @@ -439,7 +445,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].scriptPubKey = CScript() << OP_1; tx.nLockTime = 0; hash = tx.GetHash(); - mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(CheckFinalTx(CTransaction(tx), flags)); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, &prevheights, CreateBlockIndex(::ChainActive().Tip()->nHeight + 2))); // Sequence locks pass on 2nd block @@ -449,7 +455,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | (((::ChainActive().Tip()->GetMedianTimePast()+1-::ChainActive()[1]->GetMedianTimePast()) >> CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) + 1); // txFirst[1] is the 3rd block prevheights[0] = baseheight + 2; hash = tx.GetHash(); - mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); BOOST_CHECK(CheckFinalTx(CTransaction(tx), flags)); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks fail @@ -465,7 +471,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights[0] = baseheight + 3; tx.nLockTime = ::ChainActive().Tip()->nHeight + 1; hash = tx.GetHash(); - mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); BOOST_CHECK(!CheckFinalTx(CTransaction(tx), flags)); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), ::ChainActive().Tip()->nHeight + 2, ::ChainActive().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block @@ -476,7 +482,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) prevheights.resize(1); prevheights[0] = baseheight + 4; hash = tx.GetHash(); - mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); + m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); BOOST_CHECK(!CheckFinalTx(CTransaction(tx), flags)); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction(tx), flags)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), ::ChainActive().Tip()->nHeight + 2, ::ChainActive().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later @@ -513,7 +519,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) ::ChainActive().Tip()->nHeight--; SetMockTime(0); - mempool.clear(); + m_node.mempool->clear(); TestPackageSelection(chainparams, scriptPubKey, txFirst); diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 7c60abb93f..97a918da45 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -11,7 +11,7 @@ #include <script/signingprovider.h> #include <tinyformat.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index fed65afdbf..cb1ef5dcf3 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -5,7 +5,7 @@ #include <addrdb.h> #include <addrman.h> #include <clientversion.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> #include <serialize.h> @@ -99,6 +99,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); + BOOST_CHECK(Lookup(std::string("250.7.3.3", 9), addr3, 9999, false)); + BOOST_CHECK(!Lookup(std::string("250.7.3.3\0example.com", 21), addr3, 9999, false)); // Add three addresses to new table. CService source; @@ -128,9 +130,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); CAddrMan addrman2; - CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); - BOOST_CHECK(adb.Read(addrman2, ssPeers2)); + BOOST_CHECK(CAddrDB::Read(addrman2, ssPeers2)); BOOST_CHECK(addrman2.size() == 3); } @@ -160,9 +161,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); CAddrMan addrman2; - CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); - BOOST_CHECK(!adb.Read(addrman2, ssPeers2)); + BOOST_CHECK(!CAddrDB::Read(addrman2, ssPeers2)); BOOST_CHECK(addrman2.size() == 0); } @@ -303,5 +303,19 @@ BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) BOOST_CHECK_EQUAL(IsLocal(addr), false); } +BOOST_AUTO_TEST_CASE(PoissonNextSend) +{ + g_mock_deterministic_tests = true; + + int64_t now = 5000; + int average_interval_seconds = 600; + + auto poisson = ::PoissonNextSend(now, average_interval_seconds); + std::chrono::microseconds poisson_chrono = ::PoissonNextSend(std::chrono::microseconds{now}, std::chrono::seconds{average_interval_seconds}); + + BOOST_CHECK_EQUAL(poisson, poisson_chrono.count()); + + g_mock_deterministic_tests = false; +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index a3d0831624..9730b40580 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -4,7 +4,7 @@ #include <netbase.h> #include <net_permissions.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <string> @@ -13,21 +13,21 @@ BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) -static CNetAddr ResolveIP(const char* ip) +static CNetAddr ResolveIP(const std::string& ip) { CNetAddr addr; LookupHost(ip, addr, false); return addr; } -static CSubNet ResolveSubNet(const char* subnet) +static CSubNet ResolveSubNet(const std::string& subnet) { CSubNet ret; LookupSubNet(subnet, ret); return ret; } -static CNetAddr CreateInternal(const char* host) +static CNetAddr CreateInternal(const std::string& host) { CNetAddr addr; addr.SetInternal(host); @@ -54,6 +54,8 @@ BOOST_AUTO_TEST_CASE(netbase_properties) BOOST_CHECK(ResolveIP("10.0.0.1").IsRFC1918()); BOOST_CHECK(ResolveIP("192.168.1.1").IsRFC1918()); BOOST_CHECK(ResolveIP("172.31.255.255").IsRFC1918()); + BOOST_CHECK(ResolveIP("198.18.0.0").IsRFC2544()); + BOOST_CHECK(ResolveIP("198.19.255.255").IsRFC2544()); BOOST_CHECK(ResolveIP("2001:0DB8::").IsRFC3849()); BOOST_CHECK(ResolveIP("169.254.1.1").IsRFC3927()); BOOST_CHECK(ResolveIP("2002::1").IsRFC3964()); @@ -103,7 +105,7 @@ BOOST_AUTO_TEST_CASE(netbase_splithost) bool static TestParse(std::string src, std::string canon) { - CService addr(LookupNumeric(src.c_str(), 65535)); + CService addr(LookupNumeric(src, 65535)); return canon == addr.ToString(); } @@ -125,7 +127,6 @@ BOOST_AUTO_TEST_CASE(netbase_lookupnumeric) BOOST_AUTO_TEST_CASE(onioncat_test) { - // values from https://web.archive.org/web/20121122003543/http://www.cypherpunk.at/onioncat/wiki/OnionCat CNetAddr addr1(ResolveIP("5wyqrzbvrdsumnok.onion")); CNetAddr addr2(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca")); @@ -285,23 +286,23 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup() == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + std::vector<bool> asmap; // use /16 + BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // Local -> !Routable() + BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // !Valid -> !Routable() + BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() + BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() + BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor + BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector<unsigned char> internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup() == internal_group); + BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_parsenetwork) @@ -400,4 +401,22 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(std::find(strings.begin(), strings.end(), "mempool") != strings.end()); } +BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) +{ + CNetAddr addr; + BOOST_CHECK(LookupHost(std::string("127.0.0.1", 9), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0", 10), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com", 21), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com\0", 22), addr, false)); + CSubNet ret; + BOOST_CHECK(LookupSubNet(std::string("1.2.3.0/24", 10), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); + BOOST_CHECK(LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index c5513ae9fa..bf58bd63b9 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -9,7 +9,7 @@ #include <uint256.h> #include <arith_uint256.h> #include <version.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -30,7 +30,6 @@ BOOST_FIXTURE_TEST_SUITE(pmt_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(pmt_test1) { - SeedInsecureRand(false); static const unsigned int nTxCounts[] = {1, 4, 7, 17, 56, 100, 127, 256, 312, 513, 1000, 4095}; for (int i = 0; i < 12; i++) { diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 5368f82ffe..025e2b78ca 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -8,7 +8,7 @@ #include <uint256.h> #include <util/time.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 6c99021d97..0f9872f434 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -5,7 +5,7 @@ #include <chain.h> #include <chainparams.h> #include <pow.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index fc1f946bba..9782b78f2c 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -9,7 +9,7 @@ #include <serialize.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 41ca8029e5..04bf7c20c1 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -12,7 +12,7 @@ #include <support/events.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index e6fbe2355d..e0df41a971 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -4,7 +4,7 @@ #include <random.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 69db9dcf4e..532fe143ae 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <reverselock.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 5ae0812243..84a3980b19 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -7,9 +7,9 @@ #include <rpc/util.h> #include <core_io.h> -#include <init.h> #include <interfaces/chain.h> -#include <test/setup_common.h> +#include <node/context.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <boost/algorithm/string.hpp> @@ -112,14 +112,10 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign) std::string notsigned = r.get_str(); std::string privkey1 = "\"KzsXybp9jX64P5ekX1KUxRQ79Jht9uzW7LorgwE65i5rWACL6LQe\""; std::string privkey2 = "\"Kyhdf5LuKTRx4ge69ybABsiUAWjVRK4XGxAKk2FQLp2HjGMy87Z4\""; - InitInterfaces interfaces; - interfaces.chain = interfaces::MakeChain(); - g_rpc_interfaces = &interfaces; r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" [] "+prevout); BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == false); r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" ["+privkey1+","+privkey2+"] "+prevout); BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == true); - g_rpc_interfaces = nullptr; } BOOST_AUTO_TEST_CASE(rpc_createraw_op_return) diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 891aa8e5c3..4d50845256 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -4,7 +4,7 @@ #include <compat/sanity.h> #include <key.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 42242b962b..b292d5b0d0 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -5,7 +5,7 @@ #include <random.h> #include <scheduler.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/thread.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index f451d80984..8c1e843b0b 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -11,7 +11,7 @@ #include <policy/settings.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -209,20 +209,21 @@ BOOST_AUTO_TEST_CASE(is) p2sh << OP_HASH160 << ToByteVector(dummy) << OP_EQUAL; BOOST_CHECK(p2sh.IsPayToScriptHash()); - // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes: std::vector<unsigned char> direct = {OP_HASH160, 20}; direct.insert(direct.end(), 20, 0); direct.push_back(OP_EQUAL); BOOST_CHECK(CScript(direct.begin(), direct.end()).IsPayToScriptHash()); + + // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes: std::vector<unsigned char> pushdata1 = {OP_HASH160, OP_PUSHDATA1, 20}; pushdata1.insert(pushdata1.end(), 20, 0); pushdata1.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata1.begin(), pushdata1.end()).IsPayToScriptHash()); - std::vector<unsigned char> pushdata2 = {OP_HASH160, 20, 0}; + std::vector<unsigned char> pushdata2 = {OP_HASH160, OP_PUSHDATA2, 20, 0}; pushdata2.insert(pushdata2.end(), 20, 0); pushdata2.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata2.begin(), pushdata2.end()).IsPayToScriptHash()); - std::vector<unsigned char> pushdata4 = {OP_HASH160, 20, 0, 0, 0}; + std::vector<unsigned char> pushdata4 = {OP_HASH160, OP_PUSHDATA4, 20, 0, 0, 0}; pushdata4.insert(pushdata4.end(), 20, 0); pushdata4.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata4.begin(), pushdata4.end()).IsPayToScriptHash()); diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 412a57dd9d..de990d9254 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -6,7 +6,7 @@ #include <script/script.h> #include <script/signingprovider.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index caa99805c3..26015ca4c2 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -12,8 +12,8 @@ #include <script/signingprovider.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/lib/transaction_utils.h> -#include <test/setup_common.h> +#include <test/util/transaction_utils.h> +#include <test/util/setup_common.h> #include <rpc/util.h> #include <streams.h> diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 2c89a18331..9f928827cb 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index e7916f5000..40a6f69668 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -4,7 +4,7 @@ #include <test/scriptnum10.h> #include <script/script.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <limits.h> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index b90be15fba..303bb9b88c 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -5,7 +5,7 @@ #include <serialize.h> #include <streams.h> #include <hash.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <stdint.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp new file mode 100644 index 0000000000..45644834a5 --- /dev/null +++ b/src/test/settings_tests.cpp @@ -0,0 +1,177 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/settings.h> + +#include <test/util/setup_common.h> +#include <test/util/str.h> + + +#include <boost/test/unit_test.hpp> +#include <univalue.h> +#include <util/strencodings.h> +#include <vector> + +BOOST_FIXTURE_TEST_SUITE(settings_tests, BasicTestingSetup) + +//! Check settings struct contents against expected json strings. +static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) +{ + util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false); + util::SettingsValue list_value(util::SettingsValue::VARR); + for (const auto& item : GetSettingsList(settings, "section", "name", false)) { + list_value.push_back(item); + } + BOOST_CHECK_EQUAL(single_value.write().c_str(), single_val); + BOOST_CHECK_EQUAL(list_value.write().c_str(), list_val); +}; + +// Simple settings merge test case. +BOOST_AUTO_TEST_CASE(Simple) +{ + util::Settings settings; + settings.command_line_options["name"].push_back("val1"); + settings.command_line_options["name"].push_back("val2"); + settings.ro_config["section"]["name"].push_back(2); + + // The last given arg takes precedence when specified via commandline. + CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); + + util::Settings settings2; + settings2.ro_config["section"]["name"].push_back("val2"); + settings2.ro_config["section"]["name"].push_back("val3"); + + // The first given arg takes precedence when specified via config file. + CheckValues(settings2, R"("val2")", R"(["val2","val3"])"); +} + +// Confirm that a high priority setting overrides a lower priority setting even +// if the high priority setting is null. This behavior is useful for a high +// priority setting source to be able to effectively reset any setting back to +// its default value. +BOOST_AUTO_TEST_CASE(NullOverride) +{ + util::Settings settings; + settings.command_line_options["name"].push_back("value"); + BOOST_CHECK_EQUAL(R"("value")", GetSetting(settings, "section", "name", false, false).write().c_str()); + settings.forced_settings["name"] = {}; + BOOST_CHECK_EQUAL(R"(null)", GetSetting(settings, "section", "name", false, false).write().c_str()); +} + +// Test different ways settings can be merged, and verify results. This test can +// be used to confirm that updates to settings code don't change behavior +// unintentionally. +struct MergeTestingSetup : public BasicTestingSetup { + //! Max number of actions to sequence together. Can decrease this when + //! debugging to make test results easier to understand. + static constexpr int MAX_ACTIONS = 3; + + enum Action { END, SET, NEGATE, SECTION_SET, SECTION_NEGATE }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ActionList arg_actions = {}; + // command_line_options do not have sections. Only iterate over SET and NEGATE + ForEachNoDup(arg_actions, SET, NEGATE, [&]{ + ActionList conf_actions = {}; + ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&]{ + for (bool force_set : {false, true}) { + for (bool ignore_default_section_config : {false, true}) { + fn(arg_actions, conf_actions, force_set, ignore_default_section_config); + } + } + }); + }); + } +}; + +// Regression test covering different ways config settings can be merged. The +// test parses and merges settings, representing the results as strings that get +// compared against an expected hash. To debug, the result strings can be dumped +// to a file (see comments below). +BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("SETTINGS_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + const std::string& network = CBaseChainParams::MAIN; + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool force_set, + bool ignore_default_section_config) { + std::string desc; + int value_suffix = 0; + util::Settings settings; + + const std::string& name = ignore_default_section_config ? "wallet" : "server"; + auto push_values = [&](Action action, const char* value_prefix, const std::string& name_prefix, + std::vector<util::SettingsValue>& dest) { + if (action == SET || action == SECTION_SET) { + for (int i = 0; i < 2; ++i) { + dest.push_back(value_prefix + std::to_string(++value_suffix)); + desc += " " + name_prefix + name + "=" + dest.back().get_str(); + } + } else if (action == NEGATE || action == SECTION_NEGATE) { + dest.push_back(false); + desc += " " + name_prefix + "no" + name; + } + }; + + if (force_set) { + settings.forced_settings[name] = "forced"; + desc += " " + name + "=forced"; + } + for (Action arg_action : arg_actions) { + push_values(arg_action, "a", "-", settings.command_line_options[name]); + } + for (Action conf_action : conf_actions) { + bool use_section = conf_action == SECTION_SET || conf_action == SECTION_NEGATE; + push_values(conf_action, "c", use_section ? network + "." : "", + settings.ro_config[use_section ? network : ""][name]); + } + + desc += " || "; + desc += GetSetting(settings, network, name, ignore_default_section_config, /* get_chain_name= */ false).write(); + desc += " |"; + for (const auto& s : GetSettingsList(settings, network, name, ignore_default_section_config)) { + desc += " "; + desc += s.write(); + } + desc += " |"; + if (OnlyHasDefaultSectionSetting(settings, network, name)) desc += " ignored"; + desc += "\n"; + + out_sha.Write((const unsigned char*)desc.data(), desc.size()); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + + // If check below fails, should manually dump the results with: + // + // SETTINGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=settings_tests/Merge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || GetSetting() | GetSettingsList() | OnlyHasDefaultSectionSetting() + BOOST_CHECK_EQUAL(out_sha_hex, "79db02d74e3e193196541b67c068b40ebd0c124a24b3ecbe9cbf7e85b1c4ba7a"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 15f8db899b..bcc4a46873 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -10,7 +10,7 @@ #include <script/script.h> #include <serialize.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <util/strencodings.h> #include <version.h> @@ -26,10 +26,9 @@ extern UniValue read_json(const std::string& jsondata); // Old script.cpp SignatureHash function uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, unsigned int nIn, int nHashType) { - static const uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); if (nIn >= txTo.vin.size()) { - return one; + return UINT256_ONE(); } CMutableTransaction txTmp(txTo); @@ -59,7 +58,7 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un unsigned int nOut = nIn; if (nOut >= txTmp.vout.size()) { - return one; + return UINT256_ONE(); } txTmp.vout.resize(nOut+1); for (unsigned int i = 0; i < nOut; i++) @@ -119,8 +118,6 @@ BOOST_FIXTURE_TEST_SUITE(sighash_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(sighash_test) { - SeedInsecureRand(false); - #if defined(PRINT_SIGHASH_JSON) std::cout << "[\n"; std::cout << "\t[\"raw_transaction, script, input_index, hashType, signature_hash (result)\"],\n"; @@ -193,7 +190,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) CDataStream stream(ParseHex(raw_tx), SER_NETWORK, PROTOCOL_VERSION); stream >> tx; - CValidationState state; + TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); BOOST_CHECK(state.IsValid()); diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index a32f2cda92..6462fcefe3 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -9,7 +9,7 @@ #include <script/script.h> #include <script/standard.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 1cba3a1297..7ede79279f 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 6075fbfeca..177d8fda73 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -338,7 +338,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) { // Make this test deterministic. - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); for (int rep = 0; rep < 50; ++rep) { FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index c1399d2dbe..5c6c2ee38e 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <sync.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -38,7 +38,7 @@ BOOST_AUTO_TEST_CASE(potential_deadlock_detected) g_debug_lockorder_abort = false; #endif - CCriticalSection rmutex1, rmutex2; + RecursiveMutex rmutex1, rmutex2; TestPotentialDeadLockDetected(rmutex1, rmutex2); Mutex mutex1, mutex2; diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 7b00222ab7..19bd0d142f 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -5,7 +5,8 @@ #include <netaddress.h> #include <noui.h> -#include <test/setup_common.h> +#include <test/util/logging.h> +#include <test/util/setup_common.h> #include <timedata.h> #include <warnings.h> @@ -59,11 +60,12 @@ BOOST_AUTO_TEST_CASE(addtimedata) MultiAddTimeData(3, DEFAULT_MAX_TIME_ADJUSTMENT + 1); // Filter size is 1 + 3 = 4: It is always initialized with a single element (offset 0) - noui_suppress(); - MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 - noui_reconnect(); + { + ASSERT_DEBUG_LOG("Please check that your computer's date and time are correct!"); + MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 + } - BOOST_CHECK(GetWarnings("gui").find("clock is wrong") != std::string::npos); + BOOST_CHECK(GetWarnings(true).find("clock is wrong") != std::string::npos); // nTimeOffset is not changed if the median of offsets exceeds DEFAULT_MAX_TIME_ADJUSTMENT BOOST_CHECK_EQUAL(GetTimeOffset(), 0); diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp index d846062d9b..41aa17988c 100644 --- a/src/test/torcontrol_tests.cpp +++ b/src/test/torcontrol_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 34192c6b6a..fb45ce0ee6 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -4,7 +4,7 @@ #include <test/data/tx_invalid.json.h> #include <test/data/tx_valid.json.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <clientversion.h> #include <checkqueue.h> @@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); CTransaction tx(deserialize, stream); - CValidationState state; + TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); BOOST_CHECK(state.IsValid()); @@ -239,7 +239,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); CTransaction tx(deserialize, stream); - CValidationState state; + TxValidationState state; fValid = CheckTransaction(tx, state) && state.IsValid(); PrecomputedTransactionData txdata(tx); @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_CASE(basic_transaction_tests) CDataStream stream(vch, SER_DISK, CLIENT_VERSION); CMutableTransaction tx; stream >> tx; - CValidationState state; + TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(CTransaction(tx), state) && state.IsValid(), "Simple deserialized transaction should be valid."); // Check that duplicate txins fail @@ -706,24 +706,53 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) BOOST_CHECK_EQUAL(nDustThreshold, 546); // dust: t.vout[0].nValue = nDustThreshold - 1; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "dust"); // not dust: t.vout[0].nValue = nDustThreshold; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + // Disallowed nVersion + t.nVersion = -1; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "version"); + + t.nVersion = 0; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "version"); + + t.nVersion = 3; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "version"); + + // Allowed nVersion + t.nVersion = 1; + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + + t.nVersion = 2; + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + // Check dust with odd relay fee to verify rounding: // nDustThreshold = 182 * 3702 / 1000 dustRelayFee = CFeeRate(3702); // dust: t.vout[0].nValue = 673 - 1; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "dust"); // not dust: t.vout[0].nValue = 673; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); t.vout[0].scriptPubKey = CScript() << OP_1; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptpubkey"); // MAX_OP_RETURN_RELAY-byte TX_NULL_DATA (standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); @@ -733,7 +762,9 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // MAX_OP_RETURN_RELAY+1-byte TX_NULL_DATA (non-standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY + 1, t.vout[0].scriptPubKey.size()); + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptpubkey"); // Data payload can be encoded in any way... t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex(""); @@ -748,7 +779,9 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // ...so long as it only contains PUSHDATA's t.vout[0].scriptPubKey = CScript() << OP_RETURN << OP_RETURN; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptpubkey"); // TX_NULL_DATA w/o PUSHDATA t.vout.resize(1); @@ -759,15 +792,46 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout.resize(2); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "multi-op-return"); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "multi-op-return"); t.vout[0].scriptPubKey = CScript() << OP_RETURN; t.vout[1].scriptPubKey = CScript() << OP_RETURN; + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "multi-op-return"); + + // Check large scriptSig (non-standard if size is >1650 bytes) + t.vout.resize(1); + t.vout[0].nValue = MAX_MONEY; + t.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + // OP_PUSHDATA2 with len (3 bytes) + data (1647 bytes) = 1650 bytes + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1647, 0); // 1650 + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1648, 0); // 1651 + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptsig-size"); + + // Check bare multisig (standard if policy flag fIsBareMultisigStd is set) + fIsBareMultisigStd = true; + t.vout[0].scriptPubKey = GetScriptForMultisig(1, {key.GetPubKey()}); // simple 1-of-1 + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(65, 0); + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + + fIsBareMultisigStd = false; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "bare-multisig"); + fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 0ac4b7ebc9..4b0214a15a 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -5,7 +5,7 @@ #include <chainparams.h> #include <index/txindex.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 2356e0ccdc..cace75f093 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -6,7 +6,7 @@ #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/script.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -30,27 +30,26 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK(CTransaction(coinbaseTx).IsCoinBase()); - CValidationState state; + TxValidationState state; LOCK(cs_main); - unsigned int initialPoolSize = mempool.size(); + unsigned int initialPoolSize = m_node.mempool->size(); BOOST_CHECK_EQUAL( false, - AcceptToMemoryPool(mempool, state, MakeTransactionRef(coinbaseTx), - nullptr /* pfMissingInputs */, + AcceptToMemoryPool(*m_node.mempool, state, MakeTransactionRef(coinbaseTx), nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */)); // Check that the transaction hasn't been added to mempool. - BOOST_CHECK_EQUAL(mempool.size(), initialPoolSize); + BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); // Check that the validation state reflects the unsuccessful attempt. BOOST_CHECK(state.IsInvalid()); BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); - BOOST_CHECK(state.GetReason() == ValidationInvalidReason::CONSENSUS); + BOOST_CHECK(state.GetResult() == TxValidationResult::TX_CONSENSUS); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 193858cca9..7842594b80 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,23 +9,13 @@ #include <script/standard.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); -BOOST_AUTO_TEST_SUITE(tx_validationcache_tests) - -static bool -ToMemPool(const CMutableTransaction& tx) -{ - LOCK(cs_main); - - CValidationState state; - return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), nullptr /* pfMissingInputs */, - nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */); -} +BOOST_AUTO_TEST_SUITE(txvalidationcache_tests) BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) { @@ -35,6 +25,14 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + const auto ToMemPool = [this](const CMutableTransaction& tx) { + LOCK(cs_main); + + TxValidationState state; + return AcceptToMemoryPool(*m_node.mempool, state, MakeTransactionRef(tx), + nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */); + }; + // Create a double-spend of mature coinbase txn: std::vector<CMutableTransaction> spends; spends.resize(2); @@ -72,7 +70,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) LOCK(cs_main); BOOST_CHECK(::ChainActive().Tip()->GetBlockHash() != block.GetHash()); } - mempool.clear(); + m_node.mempool->clear(); // Test 3: ... and should be rejected if spend2 is in the memory pool BOOST_CHECK(ToMemPool(spends[1])); @@ -81,9 +79,9 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) LOCK(cs_main); BOOST_CHECK(::ChainActive().Tip()->GetBlockHash() != block.GetHash()); } - mempool.clear(); + m_node.mempool->clear(); - // Final sanity test: first spend in mempool, second in block, that's OK: + // Final sanity test: first spend in *m_node.mempool, second in block, that's OK: std::vector<CMutableTransaction> oneSpend; oneSpend.push_back(spends[0]); BOOST_CHECK(ToMemPool(spends[1])); @@ -94,11 +92,11 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) } // spends[1] should have been removed from the mempool when the // block with spends[0] is accepted: - BOOST_CHECK_EQUAL(mempool.size(), 0U); + BOOST_CHECK_EQUAL(m_node.mempool->size(), 0U); } -// Run CheckInputs (using CoinsTip()) on the given transaction, for all script -// flags. Test that CheckInputs passes for all flags that don't overlap with +// Run CheckInputScripts (using CoinsTip()) on the given transaction, for all script +// flags. Test that CheckInputScripts passes for all flags that don't overlap with // the failing_flags argument, but otherwise fails. // CHECKLOCKTIMEVERIFY and CHECKSEQUENCEVERIFY (and future NOP codes that may // get reassigned) have an interaction with DISCOURAGE_UPGRADABLE_NOPS: if @@ -114,7 +112,7 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail // If we add many more flags, this loop can get too expensive, but we can // rewrite in the future to randomly pick a set of flags to evaluate. for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { - CValidationState state; + TxValidationState state; // Filter out incompatible flag choices if ((test_flags & SCRIPT_VERIFY_CLEANSTACK)) { // CLEANSTACK requires P2SH and WITNESS, see VerifyScript() in @@ -125,8 +123,8 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail // WITNESS requires P2SH test_flags |= SCRIPT_VERIFY_P2SH; } - bool ret = CheckInputs(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, nullptr); - // CheckInputs should succeed iff test_flags doesn't intersect with + bool ret = CheckInputScripts(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, nullptr); + // CheckInputScripts should succeed iff test_flags doesn't intersect with // failing_flags bool expected_return_value = !(test_flags & failing_flags); BOOST_CHECK_EQUAL(ret, expected_return_value); @@ -135,13 +133,13 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail if (ret && add_to_cache) { // Check that we get a cache hit if the tx was valid std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK(scriptchecks.empty()); } else { // Check that we get script executions to check, if the transaction // was invalid, or we didn't add to cache. std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), tx.vin.size()); } } @@ -149,7 +147,7 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) { - // Test that passing CheckInputs with one set of script flags doesn't imply + // Test that passing CheckInputScripts with one set of script flags doesn't imply // that we would pass again with a different set of flags. { LOCK(cs_main); @@ -201,19 +199,19 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) { LOCK(cs_main); - CValidationState state; + TxValidationState state; PrecomputedTransactionData ptd_spend_tx(spend_tx); - BOOST_CHECK(!CheckInputs(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); + BOOST_CHECK(!CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); // If we call again asking for scriptchecks (as happens in // ConnectBlock), we should add a script check object for this -- we're // not caching invalidity (if that changes, delete this test case). std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), 1U); - // Test that CheckInputs returns true iff DERSIG-enforcing flags are + // Test that CheckInputScripts returns true iff DERSIG-enforcing flags are // not present. Don't add these checks to the cache, so that we can // test later that block validation works fine in the absence of cached // successes. @@ -270,9 +268,9 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; - CValidationState state; + TxValidationState state; PrecomputedTransactionData txdata(invalid_with_cltv_tx); - BOOST_CHECK(CheckInputs(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } // TEST CHECKSEQUENCEVERIFY @@ -298,14 +296,14 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; - CValidationState state; + TxValidationState state; PrecomputedTransactionData txdata(invalid_with_csv_tx); - BOOST_CHECK(CheckInputs(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } // TODO: add tests for remaining script flags - // Test that passing CheckInputs with a valid witness doesn't imply success + // Test that passing CheckInputScripts with a valid witness doesn't imply success // for the same tx with a different witness. { CMutableTransaction valid_with_witness_tx; @@ -359,15 +357,15 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Invalidate vin[1] tx.vin[1].scriptWitness.SetNull(); - CValidationState state; + TxValidationState state; PrecomputedTransactionData txdata(tx); // This transaction is now invalid under segwit, because of the second input. - BOOST_CHECK(!CheckInputs(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); + BOOST_CHECK(!CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); std::vector<CScriptCheck> scriptchecks; // Make sure this transaction was not cached (ie because the first // input was valid) - BOOST_CHECK(CheckInputs(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); // Should get 2 script checks back -- caching is on a whole-transaction basis. BOOST_CHECK_EQUAL(scriptchecks.size(), 2U); } diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 33a118c2bb..7293ecd325 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -6,7 +6,7 @@ #include <streams.h> #include <uint256.h> #include <version.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <sstream> diff --git a/src/test/util.cpp b/src/test/util.cpp deleted file mode 100644 index b7bb6deeaa..0000000000 --- a/src/test/util.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2019 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <test/util.h> - -#include <chainparams.h> -#include <consensus/merkle.h> -#include <key_io.h> -#include <miner.h> -#include <outputtype.h> -#include <pow.h> -#include <script/standard.h> -#include <validation.h> -#include <validationinterface.h> -#ifdef ENABLE_WALLET -#include <wallet/wallet.h> -#endif - -const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; - -#ifdef ENABLE_WALLET -std::string getnewaddress(CWallet& w) -{ - constexpr auto output_type = OutputType::BECH32; - CTxDestination dest; - std::string error; - if (!w.GetNewDestination(output_type, "", dest, error)) assert(false); - - return EncodeDestination(dest); -} - -void importaddress(CWallet& wallet, const std::string& address) -{ - LOCK(wallet.cs_wallet); - const auto dest = DecodeDestination(address); - assert(IsValidDestination(dest)); - const auto script = GetScriptForDestination(dest); - wallet.MarkDirty(); - assert(!wallet.HaveWatchOnly(script)); - if (!wallet.AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); - wallet.SetAddressBook(dest, /* label */ "", "receive"); -} -#endif // ENABLE_WALLET - -CTxIn generatetoaddress(const std::string& address) -{ - const auto dest = DecodeDestination(address); - assert(IsValidDestination(dest)); - const auto coinbase_script = GetScriptForDestination(dest); - - return MineBlock(coinbase_script); -} - -CTxIn MineBlock(const CScript& coinbase_scriptPubKey) -{ - auto block = PrepareBlock(coinbase_scriptPubKey); - - while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { - ++block->nNonce; - assert(block->nNonce); - } - - bool processed{ProcessNewBlock(Params(), block, true, nullptr)}; - assert(processed); - - return CTxIn{block->vtx[0]->GetHash(), 0}; -} - -std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey) -{ - auto block = std::make_shared<CBlock>( - BlockAssembler{Params()} - .CreateNewBlock(coinbase_scriptPubKey) - ->block); - - LOCK(cs_main); - block->nTime = ::ChainActive().Tip()->GetMedianTimePast() + 1; - block->hashMerkleRoot = BlockMerkleRoot(*block); - - return block; -} diff --git a/src/test/util/README.md b/src/test/util/README.md new file mode 100644 index 0000000000..36ad645201 --- /dev/null +++ b/src/test/util/README.md @@ -0,0 +1,11 @@ +# Test library + +This contains files for the test library, which is used by the test binaries (unit tests, benchmarks, fuzzers, gui +tests). + +Generally, the files in this folder should be well-separated modules. New code should be added to existing modules or +(when in doubt) a new module should be created. + +The utilities in here are compiled into a library, which does not hold any state. However, the main file `setup_common` +defines the common test setup for all test binaries. The test binaries will handle the global state when they +instantiate the `BasicTestingSetup` (or one of its derived classes). diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp new file mode 100644 index 0000000000..bccff5e5a6 --- /dev/null +++ b/src/test/util/blockfilter.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/blockfilter.h> + +#include <chainparams.h> +#include <validation.h> + + +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) +{ + CBlock block; + if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + return false; + } + + CBlockUndo block_undo; + if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + return false; + } + + filter = BlockFilter(filter_type, block, block_undo); + return true; +} + diff --git a/src/test/util/blockfilter.h b/src/test/util/blockfilter.h new file mode 100644 index 0000000000..79d11dcad8 --- /dev/null +++ b/src/test/util/blockfilter.h @@ -0,0 +1,13 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_BLOCKFILTER_H +#define BITCOIN_TEST_UTIL_BLOCKFILTER_H + +#include <blockfilter.h> +class CBlockIndex; + +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter); + +#endif // BITCOIN_TEST_UTIL_BLOCKFILTER_H diff --git a/src/test/util/logging.cpp b/src/test/util/logging.cpp new file mode 100644 index 0000000000..fe2e69104b --- /dev/null +++ b/src/test/util/logging.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/logging.h> + +#include <logging.h> +#include <noui.h> +#include <tinyformat.h> +#include <util/memory.h> + +#include <stdexcept> + +DebugLogHelper::DebugLogHelper(std::string message) + : m_message{std::move(message)} +{ + m_print_connection = LogInstance().PushBackCallback( + [this](const std::string& s) { + if (m_found) return; + m_found = s.find(m_message) != std::string::npos; + }); + noui_test_redirect(); +} + +void DebugLogHelper::check_found() +{ + noui_reconnect(); + LogInstance().DeleteCallback(m_print_connection); + if (!m_found) { + throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); + } +} diff --git a/src/test/util/logging.h b/src/test/util/logging.h new file mode 100644 index 0000000000..45ec44173c --- /dev/null +++ b/src/test/util/logging.h @@ -0,0 +1,29 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_LOGGING_H +#define BITCOIN_TEST_UTIL_LOGGING_H + +#include <util/macros.h> + +#include <functional> +#include <list> +#include <string> + +class DebugLogHelper +{ + const std::string m_message; + bool m_found{false}; + std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + + void check_found(); + +public: + DebugLogHelper(std::string message); + ~DebugLogHelper() { check_found(); } +}; + +#define ASSERT_DEBUG_LOG(message) DebugLogHelper PASTE2(debugloghelper, __COUNTER__)(message) + +#endif // BITCOIN_TEST_UTIL_LOGGING_H diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp new file mode 100644 index 0000000000..1df6844062 --- /dev/null +++ b/src/test/util/mining.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/mining.h> + +#include <chainparams.h> +#include <consensus/merkle.h> +#include <key_io.h> +#include <miner.h> +#include <node/context.h> +#include <pow.h> +#include <script/standard.h> +#include <validation.h> + +CTxIn generatetoaddress(const NodeContext& node, const std::string& address) +{ + const auto dest = DecodeDestination(address); + assert(IsValidDestination(dest)); + const auto coinbase_script = GetScriptForDestination(dest); + + return MineBlock(node, coinbase_script); +} + +CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +{ + auto block = PrepareBlock(node, coinbase_scriptPubKey); + + while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { + ++block->nNonce; + assert(block->nNonce); + } + + bool processed{ProcessNewBlock(Params(), block, true, nullptr)}; + assert(processed); + + return CTxIn{block->vtx[0]->GetHash(), 0}; +} + +std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +{ + assert(node.mempool); + auto block = std::make_shared<CBlock>( + BlockAssembler{*node.mempool, Params()} + .CreateNewBlock(coinbase_scriptPubKey) + ->block); + + LOCK(cs_main); + block->nTime = ::ChainActive().Tip()->GetMedianTimePast() + 1; + block->hashMerkleRoot = BlockMerkleRoot(*block); + + return block; +} diff --git a/src/test/util/mining.h b/src/test/util/mining.h new file mode 100644 index 0000000000..5f250fffe8 --- /dev/null +++ b/src/test/util/mining.h @@ -0,0 +1,25 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_MINING_H +#define BITCOIN_TEST_UTIL_MINING_H + +#include <memory> +#include <string> + +class CBlock; +class CScript; +class CTxIn; +struct NodeContext; + +/** Returns the generated coin */ +CTxIn MineBlock(const NodeContext&, const CScript& coinbase_scriptPubKey); + +/** Prepare a block to be mined */ +std::shared_ptr<CBlock> PrepareBlock(const NodeContext&, const CScript& coinbase_scriptPubKey); + +/** RPC-like helper function, returns the generated coin */ +CTxIn generatetoaddress(const NodeContext&, const std::string& address); + +#endif // BITCOIN_TEST_UTIL_MINING_H diff --git a/src/test/setup_common.cpp b/src/test/util/setup_common.cpp index bbdf1ef830..ccb3064d59 100644 --- a/src/test/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <banman.h> #include <chainparams.h> @@ -15,6 +15,7 @@ #include <net.h> #include <noui.h> #include <pow.h> +#include <rpc/blockchain.h> #include <rpc/register.h> #include <rpc/server.h> #include <script/sigcache.h> @@ -33,6 +34,27 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; FastRandomContext g_insecure_rand_ctx; +/** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ +static FastRandomContext g_insecure_rand_ctx_temp_path; + +/** Return the unsigned from the environment var if available, otherwise 0 */ +static uint256 GetUintFromEnv(const std::string& env_name) +{ + const char* num = std::getenv(env_name.c_str()); + if (!num) return {}; + return uint256S(num); +} + +void Seed(FastRandomContext& ctx) +{ + // Should be enough to get the seed once for the process + static uint256 seed{}; + static const std::string RANDOM_CTX_SEED{"RANDOM_CTX_SEED"}; + if (seed.IsNull()) seed = GetUintFromEnv(RANDOM_CTX_SEED); + if (seed.IsNull()) seed = GetRandHash(); + LogPrintf("%s: Setting random seed for current tests to %s=%s\n", __func__, RANDOM_CTX_SEED, seed.GetHex()); + ctx = FastRandomContext(seed); +} std::ostream& operator<<(std::ostream& os, const uint256& num) { @@ -41,13 +63,15 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) } BasicTestingSetup::BasicTestingSetup(const std::string& chainName) - : m_path_root(fs::temp_directory_path() / "test_common_" PACKAGE_NAME / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30)))) + : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / std::to_string(g_insecure_rand_ctx_temp_path.rand32())} { fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); SelectParams(chainName); + SeedInsecureRand(); gArgs.ForceSetArg("-printtoconsole", "0"); + if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(); LogInstance().StartLogging(); SHA256AutoDetect(); @@ -76,6 +100,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework // instead of unit tests, but for now we need these here. + g_rpc_node = &m_node; RegisterAllCoreRPCCommands(tableRPC); // We have to run a scheduler thread to prevent ActivateBestChain @@ -83,7 +108,6 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - mempool.setSanityCheck(1.0); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); g_chainstate = MakeUnique<CChainState>(); ::ChainstateActive().InitCoinsDB( @@ -95,17 +119,22 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha throw std::runtime_error("LoadGenesisBlock failed."); } - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, chainparams)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); } - nScriptCheckThreads = 3; - for (int i = 0; i < nScriptCheckThreads - 1; i++) + // Start script-checking threads. Set g_parallel_script_checks to true so they are used. + constexpr int script_check_threads = 2; + for (int i = 0; i < script_check_threads; ++i) { threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); + } + g_parallel_script_checks = true; - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. + m_node.mempool = &::mempool; + m_node.mempool->setSanityCheck(1.0); + m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); + m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. } TestingSetup::~TestingSetup() @@ -114,18 +143,21 @@ TestingSetup::~TestingSetup() threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - g_connman.reset(); - g_banman.reset(); + g_rpc_node = nullptr; + m_node.connman.reset(); + m_node.banman.reset(); + m_node.mempool = nullptr; UnloadBlockIndex(); g_chainstate.reset(); pblocktree.reset(); } -TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) +TestChain100Setup::TestChain100Setup() { // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. // TODO: fix the code to support SegWit blocks. gArgs.ForceSetArg("-segwitheight", "432"); + // Need to recreate chainparams SelectParams(CBaseChainParams::REGTEST); // Generate a 100-block chain: @@ -139,15 +171,12 @@ TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) } } -// // Create a new block with just given transactions, coinbase paying to // scriptPubKey, and try to add it to the current chain. -// -CBlock -TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) +CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; // Replace mempool-selected txns with just coinbase plus passed-in txns: @@ -172,6 +201,7 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& TestChain100Setup::~TestChain100Setup() { + gArgs.ForceSetArg("-segwitheight", "0"); } diff --git a/src/test/setup_common.h b/src/test/util/setup_common.h index 6c9494898c..6741be8480 100644 --- a/src/test/setup_common.h +++ b/src/test/util/setup_common.h @@ -2,12 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_TEST_SETUP_COMMON_H -#define BITCOIN_TEST_SETUP_COMMON_H +#ifndef BITCOIN_TEST_UTIL_SETUP_COMMON_H +#define BITCOIN_TEST_UTIL_SETUP_COMMON_H #include <chainparamsbase.h> #include <fs.h> #include <key.h> +#include <node/context.h> #include <pubkey.h> #include <random.h> #include <scheduler.h> @@ -17,6 +18,9 @@ #include <boost/thread.hpp> +/** This is connected to the logger. Can be used to redirect logs to any other log */ +extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; + // Enable BOOST_CHECK_EQUAL for enum class types template <typename T> std::ostream& operator<<(typename std::enable_if<std::is_enum<T>::value, std::ostream>::type& stream, const T& e) @@ -38,9 +42,21 @@ extern FastRandomContext g_insecure_rand_ctx; */ extern bool g_mock_deterministic_tests; -static inline void SeedInsecureRand(bool deterministic = false) +enum class SeedRand { + ZEROS, //!< Seed with a compile time constant of zeros + SEED, //!< Call the Seed() helper +}; + +/** Seed the given random ctx or use the seed passed in via an environment var */ +void Seed(FastRandomContext& ctx); + +static inline void SeedInsecureRand(SeedRand seed = SeedRand::SEED) { - g_insecure_rand_ctx = FastRandomContext(deterministic); + if (seed == SeedRand::ZEROS) { + g_insecure_rand_ctx = FastRandomContext(/* deterministic */ true); + } else { + Seed(g_insecure_rand_ctx); + } } static inline uint32_t InsecureRand32() { return g_insecure_rand_ctx.rand32(); } @@ -67,6 +83,7 @@ private: * Included are coins database, script check threads setup. */ struct TestingSetup : public BasicTestingSetup { + NodeContext m_node; boost::thread_group threadGroup; CScheduler scheduler; @@ -74,6 +91,12 @@ struct TestingSetup : public BasicTestingSetup { ~TestingSetup(); }; +/** Identical to TestingSetup, but chain set to regtest */ +struct RegTestingSetup : public TestingSetup { + RegTestingSetup() + : TestingSetup{CBaseChainParams::REGTEST} {} +}; + class CBlock; struct CMutableTransaction; class CScript; @@ -82,7 +105,7 @@ class CScript; // Testing fixture that pre-creates a // 100-block REGTEST-mode block chain // -struct TestChain100Setup : public TestingSetup { +struct TestChain100Setup : public RegTestingSetup { TestChain100Setup(); // Create a new block with just given transactions, coinbase paying to diff --git a/src/test/util/str.cpp b/src/test/util/str.cpp new file mode 100644 index 0000000000..c517fe44d9 --- /dev/null +++ b/src/test/util/str.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/str.h> + +#include <cstdint> +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2) +{ + if (s1.size() != s2.size()) return false; + for (size_t i = 0; i < s1.size(); ++i) { + char c1 = s1[i]; + if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); + char c2 = s2[i]; + if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); + if (c1 != c2) return false; + } + return true; +} diff --git a/src/test/util.h b/src/test/util/str.h index f90cb0d623..ef94692df0 100644 --- a/src/test/util.h +++ b/src/test/util/str.h @@ -2,37 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_TEST_UTIL_H -#define BITCOIN_TEST_UTIL_H +#ifndef BITCOIN_TEST_UTIL_STR_H +#define BITCOIN_TEST_UTIL_STR_H -#include <memory> #include <string> -class CBlock; -class CScript; -class CTxIn; -class CWallet; - -// Constants // - -extern const std::string ADDRESS_BCRT1_UNSPENDABLE; - -// Lower-level utils // - -/** Returns the generated coin */ -CTxIn MineBlock(const CScript& coinbase_scriptPubKey); -/** Prepare a block to be mined */ -std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey); - - -// RPC-like // - -/** Import the address to the wallet */ -void importaddress(CWallet& wallet, const std::string& address); -/** Returns a new address from the wallet */ -std::string getnewaddress(CWallet& w); -/** Returns the generated coin */ -CTxIn generatetoaddress(const std::string& address); +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2); /** * Increment a string. Useful to enumerate all fixed length strings with @@ -67,4 +42,4 @@ void ForEachNoDup(CharType (&string)[StringLength], CharType min_char, CharType } } -#endif // BITCOIN_TEST_UTIL_H +#endif // BITCOIN_TEST_UTIL_STR_H diff --git a/src/test/lib/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 2619fb9006..90b78effb0 100644 --- a/src/test/lib/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/lib/transaction_utils.h> +#include <test/util/transaction_utils.h> CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) { diff --git a/src/test/lib/transaction_utils.h b/src/test/util/transaction_utils.h index 6f297ac34f..57604646e7 100644 --- a/src/test/lib/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_TEST_LIB_TRANSACTION_UTILS_H -#define BITCOIN_TEST_LIB_TRANSACTION_UTILS_H +#ifndef BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H +#define BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H #include <primitives/transaction.h> @@ -16,4 +16,4 @@ CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int n // 1 output with empty scriptPubKey, full value of referenced transaction] CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit); -#endif // BITCOIN_TEST_LIB_TRANSACTION_UTILS_H +#endif // BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp new file mode 100644 index 0000000000..fd6012e9fe --- /dev/null +++ b/src/test/util/wallet.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/wallet.h> + +#include <key_io.h> +#include <outputtype.h> +#include <script/standard.h> +#ifdef ENABLE_WALLET +#include <wallet/wallet.h> +#endif + +const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; + +#ifdef ENABLE_WALLET +std::string getnewaddress(CWallet& w) +{ + constexpr auto output_type = OutputType::BECH32; + CTxDestination dest; + std::string error; + if (!w.GetNewDestination(output_type, "", dest, error)) assert(false); + + return EncodeDestination(dest); +} + +void importaddress(CWallet& wallet, const std::string& address) +{ + auto spk_man = wallet.GetLegacyScriptPubKeyMan(); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); + const auto dest = DecodeDestination(address); + assert(IsValidDestination(dest)); + const auto script = GetScriptForDestination(dest); + wallet.MarkDirty(); + assert(!spk_man->HaveWatchOnly(script)); + if (!spk_man->AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); + wallet.SetAddressBook(dest, /* label */ "", "receive"); +} +#endif // ENABLE_WALLET diff --git a/src/test/util/wallet.h b/src/test/util/wallet.h new file mode 100644 index 0000000000..565ef1756a --- /dev/null +++ b/src/test/util/wallet.h @@ -0,0 +1,24 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_WALLET_H +#define BITCOIN_TEST_UTIL_WALLET_H + +#include <string> + +class CWallet; + +// Constants // + +extern const std::string ADDRESS_BCRT1_UNSPENDABLE; + +// RPC-like // + +/** Import the address to the wallet */ +void importaddress(CWallet& wallet, const std::string& address); +/** Returns a new address from the wallet */ +std::string getnewaddress(CWallet& w); + + +#endif // BITCOIN_TEST_UTIL_WALLET_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 02303d0f65..6f0e464891 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1,13 +1,14 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/system.h> #include <clientversion.h> +#include <optional.h> #include <sync.h> -#include <test/setup_common.h> -#include <test/util.h> +#include <test/util/setup_common.h> +#include <test/util/str.h> #include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> @@ -17,6 +18,7 @@ #include <stdint.h> #include <thread> +#include <univalue.h> #include <utility> #include <vector> #ifndef WIN32 @@ -36,7 +38,7 @@ BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_criticalsection) { - CCriticalSection cs; + RecursiveMutex cs; do { LOCK(cs); @@ -145,9 +147,17 @@ BOOST_AUTO_TEST_CASE(util_Join) BOOST_CHECK_EQUAL(Join<std::string>({"foo", "bar"}, ", ", op_upper), "FOO, BAR"); } -BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) +BOOST_AUTO_TEST_CASE(util_FormatParseISO8601DateTime) { BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(0), "1970-01-01T00:00:00Z"); + + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1970-01-01T00:00:00Z"), 0); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1960-01-01T00:00:00Z"), 0); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2011-09-30T23:36:17Z"), 1317425777); + + auto time = GetSystemTimeInSeconds(); + BOOST_CHECK_EQUAL(ParseISO8601DateTime(FormatISO8601DateTime(time)), time); } BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) @@ -158,14 +168,12 @@ BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) struct TestArgsManager : public ArgsManager { TestArgsManager() { m_network_only_args.clear(); } - std::map<std::string, std::vector<std::string> >& GetOverrideArgs() { return m_override_args; } - std::map<std::string, std::vector<std::string> >& GetConfigArgs() { return m_config_args; } void ReadConfigString(const std::string str_config) { std::istringstream streamConfig(str_config); { LOCK(cs_args); - m_config_args.clear(); + m_settings.ro_config.clear(); m_config_sections.clear(); } std::string error; @@ -182,11 +190,119 @@ struct TestArgsManager : public ArgsManager AddArg(arg.first, "", arg.second, OptionsCategory::OPTIONS); } } + using ArgsManager::GetSetting; + using ArgsManager::GetSettingsList; using ArgsManager::ReadConfigStream; using ArgsManager::cs_args; using ArgsManager::m_network; + using ArgsManager::m_settings; +}; + +//! Test GetSetting and GetArg type coercion, negation, and default value handling. +class CheckValueTest : public TestChain100Setup +{ +public: + struct Expect { + util::SettingsValue setting; + bool default_string = false; + bool default_int = false; + bool default_bool = false; + const char* string_value = nullptr; + Optional<int64_t> int_value; + Optional<bool> bool_value; + Optional<std::vector<std::string>> list_value; + const char* error = nullptr; + + Expect(util::SettingsValue s) : setting(std::move(s)) {} + Expect& DefaultString() { default_string = true; return *this; } + Expect& DefaultInt() { default_int = true; return *this; } + Expect& DefaultBool() { default_bool = true; return *this; } + Expect& String(const char* s) { string_value = s; return *this; } + Expect& Int(int64_t i) { int_value = i; return *this; } + Expect& Bool(bool b) { bool_value = b; return *this; } + Expect& List(std::vector<std::string> m) { list_value = std::move(m); return *this; } + Expect& Error(const char* e) { error = e; return *this; } + }; + + void CheckValue(unsigned int flags, const char* arg, const Expect& expect) + { + TestArgsManager test; + test.SetupArgs({{"-value", flags}}); + const char* argv[] = {"ignored", arg}; + std::string error; + bool success = test.ParseParameters(arg ? 2 : 1, (char**)argv, error); + + BOOST_CHECK_EQUAL(test.GetSetting("-value").write(), expect.setting.write()); + auto settings_list = test.GetSettingsList("-value"); + if (expect.setting.isNull() || expect.setting.isFalse()) { + BOOST_CHECK_EQUAL(settings_list.size(), 0); + } else { + BOOST_CHECK_EQUAL(settings_list.size(), 1); + BOOST_CHECK_EQUAL(settings_list[0].write(), expect.setting.write()); + } + + if (expect.error) { + BOOST_CHECK(!success); + BOOST_CHECK_NE(error.find(expect.error), std::string::npos); + } else { + BOOST_CHECK(success); + BOOST_CHECK_EQUAL(error, ""); + } + + if (expect.default_string) { + BOOST_CHECK_EQUAL(test.GetArg("-value", "zzzzz"), "zzzzz"); + } else if (expect.string_value) { + BOOST_CHECK_EQUAL(test.GetArg("-value", "zzzzz"), expect.string_value); + } else { + BOOST_CHECK(!success); + } + + if (expect.default_int) { + BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), 99999); + } else if (expect.int_value) { + BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), *expect.int_value); + } else { + BOOST_CHECK(!success); + } + + if (expect.default_bool) { + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), false); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), true); + } else if (expect.bool_value) { + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), *expect.bool_value); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), *expect.bool_value); + } else { + BOOST_CHECK(!success); + } + + if (expect.list_value) { + auto l = test.GetArgs("-value"); + BOOST_CHECK_EQUAL_COLLECTIONS(l.begin(), l.end(), expect.list_value->begin(), expect.list_value->end()); + } else { + BOOST_CHECK(!success); + } + } }; +BOOST_FIXTURE_TEST_CASE(util_CheckValue, CheckValueTest) +{ + using M = ArgsManager; + + CheckValue(M::ALLOW_ANY, nullptr, Expect{{}}.DefaultString().DefaultInt().DefaultBool().List({})); + CheckValue(M::ALLOW_ANY, "-novalue", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=0", Expect{true}.String("1").Int(1).Bool(true).List({"1"})); + CheckValue(M::ALLOW_ANY, "-novalue=1", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=2", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=abc", Expect{true}.String("1").Int(1).Bool(true).List({"1"})); + CheckValue(M::ALLOW_ANY, "-value", Expect{""}.String("").Int(0).Bool(true).List({""})); + CheckValue(M::ALLOW_ANY, "-value=", Expect{""}.String("").Int(0).Bool(true).List({""})); + CheckValue(M::ALLOW_ANY, "-value=0", Expect{"0"}.String("0").Int(0).Bool(false).List({"0"})); + CheckValue(M::ALLOW_ANY, "-value=1", Expect{"1"}.String("1").Int(1).Bool(true).List({"1"})); + CheckValue(M::ALLOW_ANY, "-value=2", Expect{"2"}.String("2").Int(2).Bool(true).List({"2"})); + CheckValue(M::ALLOW_ANY, "-value=abc", Expect{"abc"}.String("abc").Int(0).Bool(false).List({"abc"})); +} + BOOST_AUTO_TEST_CASE(util_ParseParameters) { TestArgsManager testArgs; @@ -198,44 +314,100 @@ BOOST_AUTO_TEST_CASE(util_ParseParameters) const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; std::string error; + LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, ccc, d}); BOOST_CHECK(testArgs.ParseParameters(0, (char**)argv_test, error)); - BOOST_CHECK(testArgs.GetOverrideArgs().empty() && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); - BOOST_CHECK(testArgs.GetOverrideArgs().empty() && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(testArgs.GetOverrideArgs().size() == 3 && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 3 && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); - BOOST_CHECK(testArgs.GetOverrideArgs().count("-a") && testArgs.GetOverrideArgs().count("-b") && testArgs.GetOverrideArgs().count("-ccc") - && !testArgs.GetOverrideArgs().count("f") && !testArgs.GetOverrideArgs().count("-d")); - - BOOST_CHECK(testArgs.GetOverrideArgs()["-a"].size() == 1); - BOOST_CHECK(testArgs.GetOverrideArgs()["-a"].front() == ""); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].size() == 2); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].front() == "argument"); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].back() == "multiple"); + BOOST_CHECK(testArgs.m_settings.command_line_options.count("a") && testArgs.m_settings.command_line_options.count("b") && testArgs.m_settings.command_line_options.count("ccc") + && !testArgs.m_settings.command_line_options.count("f") && !testArgs.m_settings.command_line_options.count("d")); + + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].size() == 1); + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].front().get_str() == ""); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].size() == 2); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].front().get_str() == "argument"); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].back().get_str() == "multiple"); BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } +static void TestParse(const std::string& str, bool expected_bool, int64_t expected_int) +{ + TestArgsManager test; + test.SetupArgs({{"-value", ArgsManager::ALLOW_ANY}}); + std::string arg = "-value=" + str; + const char* argv[] = {"ignored", arg.c_str()}; + std::string error; + BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); + BOOST_CHECK_EQUAL(test.GetArg("-value", 99998), expected_int); + BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), expected_int); +} + +// Test bool and int parsing. +BOOST_AUTO_TEST_CASE(util_ArgParsing) +{ + // Some of these cases could be ambiguous or surprising to users, and might + // be worth triggering errors or warnings in the future. But for now basic + // test coverage is useful to avoid breaking backwards compatibility + // unintentionally. + TestParse("", true, 0); + TestParse(" ", false, 0); + TestParse("0", false, 0); + TestParse("0 ", false, 0); + TestParse(" 0", false, 0); + TestParse("+0", false, 0); + TestParse("-0", false, 0); + TestParse("5", true, 5); + TestParse("5 ", true, 5); + TestParse(" 5", true, 5); + TestParse("+5", true, 5); + TestParse("-5", true, -5); + TestParse("0 5", false, 0); + TestParse("5 0", true, 5); + TestParse("050", true, 50); + TestParse("0.", false, 0); + TestParse("5.", true, 5); + TestParse("0.0", false, 0); + TestParse("0.5", false, 0); + TestParse("5.0", true, 5); + TestParse("5.5", true, 5); + TestParse("x", false, 0); + TestParse("x0", false, 0); + TestParse("x5", false, 0); + TestParse("0x", false, 0); + TestParse("5x", true, 5); + TestParse("0x5", false, 0); + TestParse("false", false, 0); + TestParse("true", false, 0); + TestParse("yes", false, 0); + TestParse("no", false, 0); +} + BOOST_AUTO_TEST_CASE(util_GetBoolArg) { TestArgsManager testArgs; - const auto a = std::make_pair("-a", ArgsManager::ALLOW_BOOL); - const auto b = std::make_pair("-b", ArgsManager::ALLOW_BOOL); - const auto c = std::make_pair("-c", ArgsManager::ALLOW_BOOL); - const auto d = std::make_pair("-d", ArgsManager::ALLOW_BOOL); - const auto e = std::make_pair("-e", ArgsManager::ALLOW_BOOL); - const auto f = std::make_pair("-f", ArgsManager::ALLOW_BOOL); + const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); + const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); + const auto c = std::make_pair("-c", ArgsManager::ALLOW_ANY); + const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); + const auto e = std::make_pair("-e", ArgsManager::ALLOW_ANY); + const auto f = std::make_pair("-f", ArgsManager::ALLOW_ANY); const char *argv_test[] = { "ignored", "-a", "-nob", "-c=0", "-d=1", "-e=false", "-f=true"}; std::string error; + LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, c, d, e, f}); BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); @@ -244,8 +416,8 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArg) BOOST_CHECK(testArgs.IsArgSet({'-', opt}) || !opt); // Nothing else should be in the map - BOOST_CHECK(testArgs.GetOverrideArgs().size() == 6 && - testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 6 && + testArgs.m_settings.ro_config.empty()); // The -no prefix should get stripped on the way in. BOOST_CHECK(!testArgs.IsArgSet("-nob")); @@ -269,8 +441,8 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArgEdgeCases) TestArgsManager testArgs; // Params test - const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_BOOL); - const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_BOOL); + const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); + const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); const char *argv_test[] = {"ignored", "-nofoo", "-foo", "-nobar=0"}; testArgs.SetupArgs({foo, bar}); std::string error; @@ -341,38 +513,42 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) "iii=2\n"; TestArgsManager test_args; - const auto a = std::make_pair("-a", ArgsManager::ALLOW_BOOL); - const auto b = std::make_pair("-b", ArgsManager::ALLOW_BOOL); - const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_STRING); - const auto d = std::make_pair("-d", ArgsManager::ALLOW_STRING); + LOCK(test_args.cs_args); + const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); + const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); + const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_ANY); + const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); const auto e = std::make_pair("-e", ArgsManager::ALLOW_ANY); - const auto fff = std::make_pair("-fff", ArgsManager::ALLOW_BOOL); - const auto ggg = std::make_pair("-ggg", ArgsManager::ALLOW_BOOL); - const auto h = std::make_pair("-h", ArgsManager::ALLOW_BOOL); - const auto i = std::make_pair("-i", ArgsManager::ALLOW_BOOL); - const auto iii = std::make_pair("-iii", ArgsManager::ALLOW_INT); + const auto fff = std::make_pair("-fff", ArgsManager::ALLOW_ANY); + const auto ggg = std::make_pair("-ggg", ArgsManager::ALLOW_ANY); + const auto h = std::make_pair("-h", ArgsManager::ALLOW_ANY); + const auto i = std::make_pair("-i", ArgsManager::ALLOW_ANY); + const auto iii = std::make_pair("-iii", ArgsManager::ALLOW_ANY); test_args.SetupArgs({a, b, ccc, d, e, fff, ggg, h, i, iii}); test_args.ReadConfigString(str_config); // expectation: a, b, ccc, d, fff, ggg, h, i end up in map // so do sec1.ccc, sec1.d, sec1.h, sec2.ccc, sec2.iii - BOOST_CHECK(test_args.GetOverrideArgs().empty()); - BOOST_CHECK(test_args.GetConfigArgs().size() == 13); - - BOOST_CHECK(test_args.GetConfigArgs().count("-a") - && test_args.GetConfigArgs().count("-b") - && test_args.GetConfigArgs().count("-ccc") - && test_args.GetConfigArgs().count("-d") - && test_args.GetConfigArgs().count("-fff") - && test_args.GetConfigArgs().count("-ggg") - && test_args.GetConfigArgs().count("-h") - && test_args.GetConfigArgs().count("-i") + BOOST_CHECK(test_args.m_settings.command_line_options.empty()); + BOOST_CHECK(test_args.m_settings.ro_config.size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config[""].size() == 8); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].size() == 2); + + BOOST_CHECK(test_args.m_settings.ro_config[""].count("a") + && test_args.m_settings.ro_config[""].count("b") + && test_args.m_settings.ro_config[""].count("ccc") + && test_args.m_settings.ro_config[""].count("d") + && test_args.m_settings.ro_config[""].count("fff") + && test_args.m_settings.ro_config[""].count("ggg") + && test_args.m_settings.ro_config[""].count("h") + && test_args.m_settings.ro_config[""].count("i") ); - BOOST_CHECK(test_args.GetConfigArgs().count("-sec1.ccc") - && test_args.GetConfigArgs().count("-sec1.h") - && test_args.GetConfigArgs().count("-sec2.ccc") - && test_args.GetConfigArgs().count("-sec2.iii") + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc") + && test_args.m_settings.ro_config["sec1"].count("h") + && test_args.m_settings.ro_config["sec2"].count("ccc") + && test_args.m_settings.ro_config["sec2"].count("iii") ); BOOST_CHECK(test_args.IsArgSet("-a") @@ -511,24 +687,25 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) BOOST_AUTO_TEST_CASE(util_GetArg) { TestArgsManager testArgs; - testArgs.GetOverrideArgs().clear(); - testArgs.GetOverrideArgs()["strtest1"] = {"string..."}; + LOCK(testArgs.cs_args); + testArgs.m_settings.command_line_options.clear(); + testArgs.m_settings.command_line_options["strtest1"] = {"string..."}; // strtest2 undefined on purpose - testArgs.GetOverrideArgs()["inttest1"] = {"12345"}; - testArgs.GetOverrideArgs()["inttest2"] = {"81985529216486895"}; + testArgs.m_settings.command_line_options["inttest1"] = {"12345"}; + testArgs.m_settings.command_line_options["inttest2"] = {"81985529216486895"}; // inttest3 undefined on purpose - testArgs.GetOverrideArgs()["booltest1"] = {""}; + testArgs.m_settings.command_line_options["booltest1"] = {""}; // booltest2 undefined on purpose - testArgs.GetOverrideArgs()["booltest3"] = {"0"}; - testArgs.GetOverrideArgs()["booltest4"] = {"1"}; + testArgs.m_settings.command_line_options["booltest3"] = {"0"}; + testArgs.m_settings.command_line_options["booltest4"] = {"1"}; // priorities - testArgs.GetOverrideArgs()["pritest1"] = {"a", "b"}; - testArgs.GetConfigArgs()["pritest2"] = {"a", "b"}; - testArgs.GetOverrideArgs()["pritest3"] = {"a"}; - testArgs.GetConfigArgs()["pritest3"] = {"b"}; - testArgs.GetOverrideArgs()["pritest4"] = {"a","b"}; - testArgs.GetConfigArgs()["pritest4"] = {"c","d"}; + testArgs.m_settings.command_line_options["pritest1"] = {"a", "b"}; + testArgs.m_settings.ro_config[""]["pritest2"] = {"a", "b"}; + testArgs.m_settings.command_line_options["pritest3"] = {"a"}; + testArgs.m_settings.ro_config[""]["pritest3"] = {"b"}; + testArgs.m_settings.command_line_options["pritest4"] = {"a","b"}; + testArgs.m_settings.ro_config[""]["pritest4"] = {"c","d"}; BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); @@ -549,8 +726,8 @@ BOOST_AUTO_TEST_CASE(util_GetArg) BOOST_AUTO_TEST_CASE(util_GetChainName) { TestArgsManager test_args; - const auto testnet = std::make_pair("-testnet", ArgsManager::ALLOW_BOOL); - const auto regtest = std::make_pair("-regtest", ArgsManager::ALLOW_BOOL); + const auto testnet = std::make_pair("-testnet", ArgsManager::ALLOW_ANY); + const auto regtest = std::make_pair("-regtest", ArgsManager::ALLOW_ANY); test_args.SetupArgs({testnet, regtest}); const char* argv_testnet[] = {"cmd", "-testnet"}; @@ -882,6 +1059,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) desc += " "; desc += argstr + 1; conf += argstr + 1; + conf += "\n"; } std::istringstream conf_stream(conf); BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); @@ -920,7 +1098,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) // Results file is formatted like: // // <input> || <output> - BOOST_CHECK_EQUAL(out_sha_hex, "94b4ad55c8ac639a56b93e36f7e32e4c611fd7d7dd7b2be6a71707b1eadcaec7"); + BOOST_CHECK_EQUAL(out_sha_hex, "f0b3a3c29869edc765d579c928f7f1690a71fbb673b49ccf39cbc4de18156a0d"); } BOOST_AUTO_TEST_CASE(util_FormatMoney) @@ -999,6 +1177,11 @@ BOOST_AUTO_TEST_CASE(util_ParseMoney) // Parsing negative amounts must fail BOOST_CHECK(!ParseMoney("-1", ret)); + + // Parsing strings with embedded NUL characters should fail + BOOST_CHECK(!ParseMoney(std::string("\0-1", 3), ret)); + BOOST_CHECK(!ParseMoney(std::string("\01", 2), ret)); + BOOST_CHECK(!ParseMoney(std::string("1\0", 2), ret)); } BOOST_AUTO_TEST_CASE(util_IsHex) @@ -1043,7 +1226,7 @@ BOOST_AUTO_TEST_CASE(util_IsHexNumber) BOOST_AUTO_TEST_CASE(util_seed_insecure_rand) { - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); for (int mod=2;mod<11;mod++) { int mask = 1; diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 71c0168ca3..78dbf848bb 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/threadnames.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <thread> #include <vector> diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index b3368d44b6..dae389a167 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -11,20 +11,26 @@ #include <pow.h> #include <random.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <validation.h> #include <validationinterface.h> #include <thread> -struct RegtestingSetup : public TestingSetup { - RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} -}; - static const std::vector<unsigned char> V_OP_TRUE{OP_TRUE}; -BOOST_FIXTURE_TEST_SUITE(validation_block_tests, RegtestingSetup) +namespace validation_block_tests { +struct MinerTestingSetup : public RegTestingSetup { + std::shared_ptr<CBlock> Block(const uint256& prev_hash); + std::shared_ptr<const CBlock> GoodBlock(const uint256& prev_hash); + std::shared_ptr<const CBlock> BadBlock(const uint256& prev_hash); + std::shared_ptr<CBlock> FinalizeBlock(std::shared_ptr<CBlock> pblock); + void BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks); +}; +} // namespace validation_block_tests + +BOOST_FIXTURE_TEST_SUITE(validation_block_tests, MinerTestingSetup) struct TestSubscriber : public CValidationInterface { uint256 m_expected_tip; @@ -44,15 +50,16 @@ struct TestSubscriber : public CValidationInterface { m_expected_tip = block->GetHash(); } - void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override { BOOST_CHECK_EQUAL(m_expected_tip, block->GetHash()); + BOOST_CHECK_EQUAL(m_expected_tip, pindex->GetBlockHash()); m_expected_tip = block->hashPrevBlock; } }; -std::shared_ptr<CBlock> Block(const uint256& prev_hash) +std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) { static int i = 0; static uint64_t time = Params().GenesisBlock().nTime; @@ -60,7 +67,7 @@ std::shared_ptr<CBlock> Block(const uint256& prev_hash) CScript pubKey; pubKey << i++ << OP_TRUE; - auto ptemplate = BlockAssembler(Params()).CreateNewBlock(pubKey); + auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(pubKey); auto pblock = std::make_shared<CBlock>(ptemplate->block); pblock->hashPrevBlock = prev_hash; pblock->nTime = ++time; @@ -86,7 +93,7 @@ std::shared_ptr<CBlock> Block(const uint256& prev_hash) return pblock; } -std::shared_ptr<CBlock> FinalizeBlock(std::shared_ptr<CBlock> pblock) +std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> pblock) { LOCK(cs_main); // For LookupBlockIndex GenerateCoinbaseCommitment(*pblock, LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); @@ -101,13 +108,13 @@ std::shared_ptr<CBlock> FinalizeBlock(std::shared_ptr<CBlock> pblock) } // construct a valid block -std::shared_ptr<const CBlock> GoodBlock(const uint256& prev_hash) +std::shared_ptr<const CBlock> MinerTestingSetup::GoodBlock(const uint256& prev_hash) { return FinalizeBlock(Block(prev_hash)); } // construct an invalid block (but with a valid header) -std::shared_ptr<const CBlock> BadBlock(const uint256& prev_hash) +std::shared_ptr<const CBlock> MinerTestingSetup::BadBlock(const uint256& prev_hash) { auto pblock = Block(prev_hash); @@ -122,7 +129,7 @@ std::shared_ptr<const CBlock> BadBlock(const uint256& prev_hash) return ret; } -void BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks) +void MinerTestingSetup::BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks) { if (height <= 0 || blocks.size() >= max_size) return; @@ -151,7 +158,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) } bool ignored; - CValidationState state; + BlockValidationState state; std::vector<CBlockHeader> headers; std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); @@ -278,14 +285,13 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Add the txs to the tx pool { LOCK(cs_main); - CValidationState state; + TxValidationState state; std::list<CTransactionRef> plTxnReplaced; for (const auto& tx : txs) { BOOST_REQUIRE(AcceptToMemoryPool( - ::mempool, + *m_node.mempool, state, tx, - /* pfMissingInputs */ &ignored, &plTxnReplaced, /* bypass_limits */ false, /* nAbsurdFee */ 0)); @@ -294,8 +300,8 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Check that all txs are in the pool { - LOCK(::mempool.cs); - BOOST_CHECK_EQUAL(::mempool.mapTx.size(), txs.size()); + LOCK(m_node.mempool->cs); + BOOST_CHECK_EQUAL(m_node.mempool->mapTx.size(), txs.size()); } // Run a thread that simulates an RPC caller that is polling while @@ -305,8 +311,8 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // the transactions invalidated by the reorg, or none of them, and // not some intermediate amount. while (true) { - LOCK(::mempool.cs); - if (::mempool.mapTx.size() == 0) { + LOCK(m_node.mempool->cs); + if (m_node.mempool->mapTx.size() == 0) { // We are done with the reorg break; } @@ -315,7 +321,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // be atomic. So the caller assumes that the returned mempool // is consistent. That is, it has all txs that were there // before the reorg. - assert(::mempool.mapTx.size() == txs.size()); + assert(m_node.mempool->mapTx.size() == txs.size()); continue; } LOCK(cs_main); diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp new file mode 100644 index 0000000000..ab8b957f7d --- /dev/null +++ b/src/test/validation_flush_tests.cpp @@ -0,0 +1,174 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <txmempool.h> +#include <validation.h> +#include <sync.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) + +//! Test utilities for detecting when we need to flush the coins cache based +//! on estimated memory usage. +//! +//! @sa CChainState::GetCoinsCacheSizeState() +//! +BOOST_AUTO_TEST_CASE(getcoinscachesizestate) +{ + BlockManager blockman{}; + CChainState chainstate{blockman}; + chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); + WITH_LOCK(::cs_main, chainstate.InitCoinsCache()); + CTxMemPool tx_pool{}; + + constexpr bool is_64_bit = sizeof(void*) == 8; + + LOCK(::cs_main); + auto& view = chainstate.CoinsTip(); + + //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. + auto add_coin = [](CCoinsViewCache& coins_view) -> COutPoint { + Coin newcoin; + uint256 txid = InsecureRand256(); + COutPoint outp{txid, 0}; + newcoin.nHeight = 1; + newcoin.out.nValue = InsecureRand32(); + newcoin.out.scriptPubKey.assign((uint32_t)56, 1); + coins_view.AddCoin(outp, std::move(newcoin), false); + + return outp; + }; + + // The number of bytes consumed by coin's heap data, i.e. CScript + // (prevector<28, unsigned char>) when assigned 56 bytes of data per above. + // + // See also: Coin::DynamicMemoryUsage(). + constexpr int COIN_SIZE = is_64_bit ? 80 : 64; + + auto print_view_mem_usage = [](CCoinsViewCache& view) { + BOOST_TEST_MESSAGE("CCoinsViewCache memory usage: " << view.DynamicMemoryUsage()); + }; + + constexpr size_t MAX_COINS_CACHE_BYTES = 1024; + + // Without any coins in the cache, we shouldn't need to flush. + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::OK); + + // If the initial memory allocations of cacheCoins don't match these common + // cases, we can't really continue to make assertions about memory usage. + // End the test early. + if (view.DynamicMemoryUsage() != 32 && view.DynamicMemoryUsage() != 16) { + // Add a bunch of coins to see that we at least flip over to CRITICAL. + + for (int i{0}; i < 1000; ++i) { + COutPoint res = add_coin(view); + BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); + } + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::CRITICAL); + + BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); + return; + } + + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32 : 16); + + // We should be able to add COINS_UNTIL_CRITICAL coins to the cache before going CRITICAL. + // This is contingent not only on the dynamic memory usage of the Coins + // that we're adding (COIN_SIZE bytes per), but also on how much memory the + // cacheCoins (unordered_map) preallocates. + // + // I came up with the count by examining the printed memory usage of the + // CCoinsCacheView, so it's sort of arbitrary - but it shouldn't change + // unless we somehow change the way the cacheCoins map allocates memory. + // + constexpr int COINS_UNTIL_CRITICAL = is_64_bit ? 4 : 5; + + for (int i{0}; i < COINS_UNTIL_CRITICAL; ++i) { + COutPoint res = add_coin(view); + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::OK); + } + + // Adding an additional coin will push us over the edge to CRITICAL. + add_coin(view); + print_view_mem_usage(view); + + auto size_state = chainstate.GetCoinsCacheSizeState( + tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0); + + if (!is_64_bit && size_state == CoinsCacheSizeState::LARGE) { + // On 32 bit hosts, we may hit LARGE before CRITICAL. + add_coin(view); + print_view_mem_usage(view); + } + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::CRITICAL); + + // Passing non-zero max mempool usage should allow us more headroom. + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + CoinsCacheSizeState::OK); + + for (int i{0}; i < 3; ++i) { + add_coin(view); + print_view_mem_usage(view); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + CoinsCacheSizeState::OK); + } + + // Adding another coin with the additional mempool room will put us >90% + // but not yet critical. + add_coin(view); + print_view_mem_usage(view); + + // Only perform these checks on 64 bit hosts; I haven't done the math for 32. + if (is_64_bit) { + float usage_percentage = (float)view.DynamicMemoryUsage() / (MAX_COINS_CACHE_BYTES + (1 << 10)); + BOOST_TEST_MESSAGE("CoinsTip usage percentage: " << usage_percentage); + BOOST_CHECK(usage_percentage >= 0.9); + BOOST_CHECK(usage_percentage < 1); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + CoinsCacheSizeState::LARGE); + } + + // Using the default max_* values permits way more coins to be added. + for (int i{0}; i < 1000; ++i) { + add_coin(view); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool), + CoinsCacheSizeState::OK); + } + + // Flushing the view doesn't take us back to OK because cacheCoins has + // preallocated memory that doesn't get reclaimed even after flush. + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + CoinsCacheSizeState::CRITICAL); + + view.SetBestBlock(InsecureRand256()); + BOOST_CHECK(view.Flush()); + print_view_mem_usage(view); + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + CoinsCacheSizeState::CRITICAL); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index 101025d31e..3b961db52d 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -6,7 +6,7 @@ #include <net.h> #include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 0ca3a17974..7b59d539a6 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -4,7 +4,7 @@ #include <chain.h> #include <versionbits.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <chainparams.h> #include <validation.h> #include <consensus/params.h> diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h index 0654c2ab1f..2665f8a5be 100644 --- a/src/threadinterrupt.h +++ b/src/threadinterrupt.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/threadsafety.h b/src/threadsafety.h index 47e6b2ea38..bb988dfdfd 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/timedata.cpp b/src/timedata.cpp index 9458b9ae0c..942b3cb919 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,7 +16,7 @@ #include <warnings.h> -static CCriticalSection cs_nTimeOffset; +static RecursiveMutex cs_nTimeOffset; static int64_t nTimeOffset GUARDED_BY(cs_nTimeOffset) = 0; /** diff --git a/src/tinyformat.h b/src/tinyformat.h index 182f518a0b..be63f2d5d8 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -33,6 +33,7 @@ // // * Type safety and extensibility for user defined types. // * C99 printf() compatibility, to the extent possible using std::ostream +// * POSIX extension for positional arguments // * Simplicity and minimalism. A single header file to include and distribute // with your projects. // * Augment rather than replace the standard stream formatting mechanism @@ -42,7 +43,7 @@ // Main interface example usage // ---------------------------- // -// To print a date to std::cout: +// To print a date to std::cout for American usage: // // std::string weekday = "Wednesday"; // const char* month = "July"; @@ -52,6 +53,14 @@ // // tfm::printf("%s, %s %d, %.2d:%.2d\n", weekday, month, day, hour, min); // +// POSIX extension for positional arguments is available. +// The ability to rearrange formatting arguments is an important feature +// for localization because the word order may vary in different languages. +// +// Previous example for German usage. Arguments are reordered: +// +// tfm::printf("%1$s, %3$d. %2$s, %4$d:%5$.2d\n", weekday, month, day, hour, min); +// // The strange types here emphasize the type safety of the interface; it is // possible to print a std::string using the "%s" conversion, and a // size_t using the "%d" conversion. A similar result could be achieved @@ -133,12 +142,17 @@ namespace tfm = tinyformat; //------------------------------------------------------------------------------ // Implementation details. #include <algorithm> -#include <cassert> #include <iostream> #include <sstream> -#include <stdexcept> +#include <stdexcept> // Added for Bitcoin Core + +#ifndef TINYFORMAT_ASSERT +# include <cassert> +# define TINYFORMAT_ASSERT(cond) assert(cond) +#endif #ifndef TINYFORMAT_ERROR +# include <cassert> # define TINYFORMAT_ERROR(reason) assert(0 && reason) #endif @@ -149,13 +163,13 @@ namespace tfm = tinyformat; #endif #if defined(__GLIBCXX__) && __GLIBCXX__ < 20080201 -// std::showpos is broken on old libstdc++ as provided with OSX. See +// std::showpos is broken on old libstdc++ as provided with macOS. See // http://gcc.gnu.org/ml/libstdc++/2007-11/msg00075.html # define TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND #endif #ifdef __APPLE__ -// Workaround OSX linker warning: Xcode uses different default symbol +// Workaround macOS linker warning: Xcode uses different default symbol // visibilities for static libs vs executables (see issue #25) # define TINYFORMAT_HIDDEN __attribute__((visibility("hidden"))) #else @@ -164,6 +178,7 @@ namespace tfm = tinyformat; namespace tinyformat { +// Added for Bitcoin Core class format_error: public std::runtime_error { public: @@ -218,7 +233,7 @@ template<int n> struct is_wchar<wchar_t[n]> {}; template<typename T, typename fmtT, bool convertible = is_convertible<T, fmtT>::value> struct formatValueAsType { - static void invoke(std::ostream& /*out*/, const T& /*value*/) { assert(0); } + static void invoke(std::ostream& /*out*/, const T& /*value*/) { TINYFORMAT_ASSERT(0); } }; // Specialized version for types that can actually be converted to fmtT, as // indicated by the "convertible" template parameter. @@ -240,8 +255,7 @@ struct formatZeroIntegerWorkaround<T,true> { static bool invoke(std::ostream& out, const T& value) { - if (static_cast<int>(value) == 0 && out.flags() & std::ios::showpos) - { + if (static_cast<int>(value) == 0 && out.flags() & std::ios::showpos) { out << "+0"; return true; } @@ -282,7 +296,7 @@ inline void formatTruncated(std::ostream& out, const T& value, int ntrunc) inline void formatTruncated(std::ostream& out, type* value, int ntrunc) \ { \ std::streamsize len = 0; \ - while(len < ntrunc && value[len] != 0) \ + while (len < ntrunc && value[len] != 0) \ ++len; \ out.write(value, len); \ } @@ -328,15 +342,14 @@ inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, // could otherwise lead to a crash when printing a dangling (const char*). const bool canConvertToChar = detail::is_convertible<T,char>::value; const bool canConvertToVoidPtr = detail::is_convertible<T, const void*>::value; - if(canConvertToChar && *(fmtEnd-1) == 'c') + if (canConvertToChar && *(fmtEnd-1) == 'c') detail::formatValueAsType<T, char>::invoke(out, value); - else if(canConvertToVoidPtr && *(fmtEnd-1) == 'p') + else if (canConvertToVoidPtr && *(fmtEnd-1) == 'p') detail::formatValueAsType<T, const void*>::invoke(out, value); #ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND - else if(detail::formatZeroIntegerWorkaround<T>::invoke(out, value)) /**/; + else if (detail::formatZeroIntegerWorkaround<T>::invoke(out, value)) /**/; #endif - else if(ntrunc >= 0) - { + else if (ntrunc >= 0) { // Take care not to overread C strings in truncating conversions like // "%.4s" where at most 4 characters may be read. detail::formatTruncated(out, value, ntrunc); @@ -351,8 +364,7 @@ inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, \ const char* fmtEnd, int /**/, charType value) \ { \ - switch(*(fmtEnd-1)) \ - { \ + switch (*(fmtEnd-1)) { \ case 'u': case 'd': case 'i': case 'o': case 'X': case 'x': \ out << static_cast<int>(value); break; \ default: \ @@ -490,19 +502,19 @@ namespace detail { // Type-opaque holder for an argument to format(), with associated actions on // the type held as explicit function pointers. This allows FormatArg's for -// each argument to be allocated as a homogenous array inside FormatList +// each argument to be allocated as a homogeneous array inside FormatList // whereas a naive implementation based on inheritance does not. class FormatArg { public: FormatArg() - : m_value(nullptr), - m_formatImpl(nullptr), - m_toIntImpl(nullptr) - { } + : m_value(NULL), + m_formatImpl(NULL), + m_toIntImpl(NULL) + { } template<typename T> - explicit FormatArg(const T& value) + FormatArg(const T& value) : m_value(static_cast<const void*>(&value)), m_formatImpl(&formatImpl<T>), m_toIntImpl(&toIntImpl<T>) @@ -511,15 +523,15 @@ class FormatArg void format(std::ostream& out, const char* fmtBegin, const char* fmtEnd, int ntrunc) const { - assert(m_value); - assert(m_formatImpl); + TINYFORMAT_ASSERT(m_value); + TINYFORMAT_ASSERT(m_formatImpl); m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); } int toInt() const { - assert(m_value); - assert(m_toIntImpl); + TINYFORMAT_ASSERT(m_value); + TINYFORMAT_ASSERT(m_toIntImpl); return m_toIntImpl(m_value); } @@ -549,36 +561,68 @@ class FormatArg inline int parseIntAndAdvance(const char*& c) { int i = 0; - for(;*c >= '0' && *c <= '9'; ++c) + for (;*c >= '0' && *c <= '9'; ++c) i = 10*i + (*c - '0'); return i; } -// Print literal part of format string and return next format spec -// position. +// Parse width or precision `n` from format string pointer `c`, and advance it +// to the next character. If an indirection is requested with `*`, the argument +// is read from `args[argIndex]` and `argIndex` is incremented (or read +// from `args[n]` in positional mode). Returns true if one or more +// characters were read. +inline bool parseWidthOrPrecision(int& n, const char*& c, bool positionalMode, + const detail::FormatArg* args, + int& argIndex, int numArgs) +{ + if (*c >= '0' && *c <= '9') { + n = parseIntAndAdvance(c); + } + else if (*c == '*') { + ++c; + n = 0; + if (positionalMode) { + int pos = parseIntAndAdvance(c) - 1; + if (*c != '$') + TINYFORMAT_ERROR("tinyformat: Non-positional argument used after a positional one"); + if (pos >= 0 && pos < numArgs) + n = args[pos].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Positional argument out of range"); + ++c; + } + else { + if (argIndex < numArgs) + n = args[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable width or precision"); + } + } + else { + return false; + } + return true; +} + +// Print literal part of format string and return next format spec position. // -// Skips over any occurrences of '%%', printing a literal '%' to the -// output. The position of the first % character of the next -// nontrivial format spec is returned, or the end of string. +// Skips over any occurrences of '%%', printing a literal '%' to the output. +// The position of the first % character of the next nontrivial format spec is +// returned, or the end of string. inline const char* printFormatStringLiteral(std::ostream& out, const char* fmt) { const char* c = fmt; - for(;; ++c) - { - switch(*c) - { - case '\0': - out.write(fmt, c - fmt); + for (;; ++c) { + if (*c == '\0') { + out.write(fmt, c - fmt); + return c; + } + else if (*c == '%') { + out.write(fmt, c - fmt); + if (*(c+1) != '%') return c; - case '%': - out.write(fmt, c - fmt); - if(*(c+1) != '%') - return c; - // for "%%", tack trailing % onto next literal section. - fmt = ++c; - break; - default: - break; + // for "%%", tack trailing % onto next literal section. + fmt = ++c; } } } @@ -587,23 +631,43 @@ inline const char* printFormatStringLiteral(std::ostream& out, const char* fmt) // Parse a format string and set the stream state accordingly. // // The format mini-language recognized here is meant to be the one from C99, -// with the form "%[flags][width][.precision][length]type". +// with the form "%[flags][width][.precision][length]type" with POSIX +// positional arguments extension. +// +// POSIX positional arguments extension: +// Conversions can be applied to the nth argument after the format in +// the argument list, rather than to the next unused argument. In this case, +// the conversion specifier character % (see below) is replaced by the sequence +// "%n$", where n is a decimal integer in the range [1,{NL_ARGMAX}], +// giving the position of the argument in the argument list. This feature +// provides for the definition of format strings that select arguments +// in an order appropriate to specific languages. +// +// The format can contain either numbered argument conversion specifications +// (that is, "%n$" and "*m$"), or unnumbered argument conversion specifications +// (that is, % and * ), but not both. The only exception to this is that %% +// can be mixed with the "%n$" form. The results of mixing numbered and +// unnumbered argument specifications in a format string are undefined. +// When numbered argument specifications are used, specifying the Nth argument +// requires that all the leading arguments, from the first to the (N-1)th, +// are specified in the format string. +// +// In format strings containing the "%n$" form of conversion specification, +// numbered arguments in the argument list can be referenced from the format +// string as many times as required. // // Formatting options which can't be natively represented using the ostream // state are returned in spacePadPositive (for space padded positive numbers) // and ntrunc (for truncating conversions). argIndex is incremented if // necessary to pull out variable width and precision. The function returns a // pointer to the character after the end of the current format spec. -inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositive, +inline const char* streamStateFromFormat(std::ostream& out, bool& positionalMode, + bool& spacePadPositive, int& ntrunc, const char* fmtStart, - const detail::FormatArg* formatters, - int& argIndex, int numFormatters) + const detail::FormatArg* args, + int& argIndex, int numArgs) { - if(*fmtStart != '%') - { - TINYFORMAT_ERROR("tinyformat: Not enough conversion specifiers in format string"); - return fmtStart; - } + TINYFORMAT_ASSERT(*fmtStart == '%'); // Reset stream state to defaults. out.width(0); out.precision(6); @@ -616,100 +680,113 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi bool widthSet = false; int widthExtra = 0; const char* c = fmtStart + 1; - // 1) Parse flags - for(;; ++c) - { - switch(*c) - { - case '#': - out.setf(std::ios::showpoint | std::ios::showbase); - continue; - case '0': - // overridden by left alignment ('-' flag) - if(!(out.flags() & std::ios::left)) - { - // Use internal padding so that numeric values are - // formatted correctly, eg -00010 rather than 000-10 - out.fill('0'); - out.setf(std::ios::internal, std::ios::adjustfield); - } - continue; - case '-': - out.fill(' '); - out.setf(std::ios::left, std::ios::adjustfield); - continue; - case ' ': - // overridden by show positive sign, '+' flag. - if(!(out.flags() & std::ios::showpos)) - spacePadPositive = true; - continue; - case '+': - out.setf(std::ios::showpos); - spacePadPositive = false; - widthExtra = 1; - continue; - default: - break; + + // 1) Parse an argument index (if followed by '$') or a width possibly + // preceded with '0' flag. + if (*c >= '0' && *c <= '9') { + const char tmpc = *c; + int value = parseIntAndAdvance(c); + if (*c == '$') { + // value is an argument index + if (value > 0 && value <= numArgs) + argIndex = value - 1; + else + TINYFORMAT_ERROR("tinyformat: Positional argument out of range"); + ++c; + positionalMode = true; + } + else if (positionalMode) { + TINYFORMAT_ERROR("tinyformat: Non-positional argument used after a positional one"); + } + else { + if (tmpc == '0') { + // Use internal padding so that numeric values are + // formatted correctly, eg -00010 rather than 000-10 + out.fill('0'); + out.setf(std::ios::internal, std::ios::adjustfield); + } + if (value != 0) { + // Nonzero value means that we parsed width. + widthSet = true; + out.width(value); + } } - break; } - // 2) Parse width - if(*c >= '0' && *c <= '9') - { - widthSet = true; - out.width(parseIntAndAdvance(c)); + else if (positionalMode) { + TINYFORMAT_ERROR("tinyformat: Non-positional argument used after a positional one"); } - if(*c == '*') - { - widthSet = true; + // 2) Parse flags and width if we did not do it in previous step. + if (!widthSet) { + // Parse flags + for (;; ++c) { + switch (*c) { + case '#': + out.setf(std::ios::showpoint | std::ios::showbase); + continue; + case '0': + // overridden by left alignment ('-' flag) + if (!(out.flags() & std::ios::left)) { + // Use internal padding so that numeric values are + // formatted correctly, eg -00010 rather than 000-10 + out.fill('0'); + out.setf(std::ios::internal, std::ios::adjustfield); + } + continue; + case '-': + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + continue; + case ' ': + // overridden by show positive sign, '+' flag. + if (!(out.flags() & std::ios::showpos)) + spacePadPositive = true; + continue; + case '+': + out.setf(std::ios::showpos); + spacePadPositive = false; + widthExtra = 1; + continue; + default: + break; + } + break; + } + // Parse width int width = 0; - if(argIndex < numFormatters) - width = formatters[argIndex++].toInt(); - else - TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable width"); - if(width < 0) - { - // negative widths correspond to '-' flag set - out.fill(' '); - out.setf(std::ios::left, std::ios::adjustfield); - width = -width; + widthSet = parseWidthOrPrecision(width, c, positionalMode, + args, argIndex, numArgs); + if (widthSet) { + if (width < 0) { + // negative widths correspond to '-' flag set + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + width = -width; + } + out.width(width); } - out.width(width); - ++c; } // 3) Parse precision - if(*c == '.') - { + if (*c == '.') { ++c; int precision = 0; - if(*c == '*') - { - ++c; - if(argIndex < numFormatters) - precision = formatters[argIndex++].toInt(); - else - TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable precision"); - } - else - { - if(*c >= '0' && *c <= '9') - precision = parseIntAndAdvance(c); - else if(*c == '-') // negative precisions ignored, treated as zero. - parseIntAndAdvance(++c); - } - out.precision(precision); - precisionSet = true; + parseWidthOrPrecision(precision, c, positionalMode, + args, argIndex, numArgs); + // Presence of `.` indicates precision set, unless the inferred value + // was negative in which case the default is used. + precisionSet = precision >= 0; + if (precisionSet) + out.precision(precision); } // 4) Ignore any C99 length modifier - while(*c == 'l' || *c == 'h' || *c == 'L' || - *c == 'j' || *c == 'z' || *c == 't') + while (*c == 'l' || *c == 'h' || *c == 'L' || + *c == 'j' || *c == 'z' || *c == 't') { ++c; + } // 5) We're up to the conversion specifier character. // Set stream flags based on conversion specifier (thanks to the // boost::format class for forging the way here). bool intConversion = false; - switch(*c) - { + switch (*c) { case 'u': case 'd': case 'i': out.setf(std::ios::dec, std::ios::basefield); intConversion = true; @@ -738,6 +815,18 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi case 'f': out.setf(std::ios::fixed, std::ios::floatfield); break; + case 'A': + out.setf(std::ios::uppercase); + // Falls through + case 'a': +# ifdef _MSC_VER + // Workaround https://developercommunity.visualstudio.com/content/problem/520472/hexfloat-stream-output-does-not-ignore-precision-a.html + // by always setting maximum precision on MSVC to avoid precision + // loss for doubles. + out.precision(13); +# endif + out.setf(std::ios::fixed | std::ios::scientific, std::ios::floatfield); + break; case 'G': out.setf(std::ios::uppercase); // Falls through @@ -746,17 +835,13 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi // As in boost::format, let stream decide float format. out.flags(out.flags() & ~std::ios::floatfield); break; - case 'a': case 'A': - TINYFORMAT_ERROR("tinyformat: the %a and %A conversion specs " - "are not supported"); - break; case 'c': // Handled as special case inside formatValue() break; case 's': - if(precisionSet) + if (precisionSet) ntrunc = static_cast<int>(out.precision()); - // Make %s print booleans as "true" and "false" + // Make %s print Booleans as "true" and "false" out.setf(std::ios::boolalpha); break; case 'n': @@ -770,8 +855,7 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi default: break; } - if(intConversion && precisionSet && !widthSet) - { + if (intConversion && precisionSet && !widthSet) { // "precision" for integers gives the minimum number of digits (to be // padded with zeros on the left). This isn't really supported by the // iostreams, but we can approximately simulate it with the width if @@ -786,8 +870,8 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi //------------------------------------------------------------------------------ inline void formatImpl(std::ostream& out, const char* fmt, - const detail::FormatArg* formatters, - int numFormatters) + const detail::FormatArg* args, + int numArgs) { // Saved stream state std::streamsize origWidth = out.width(); @@ -795,26 +879,34 @@ inline void formatImpl(std::ostream& out, const char* fmt, std::ios::fmtflags origFlags = out.flags(); char origFill = out.fill(); - for (int argIndex = 0; argIndex < numFormatters; ++argIndex) - { - // Parse the format string + // "Positional mode" means all format specs should be of the form "%n$..." + // with `n` an integer. We detect this in `streamStateFromFormat`. + bool positionalMode = false; + int argIndex = 0; + while (true) { fmt = printFormatStringLiteral(out, fmt); + if (*fmt == '\0') { + if (!positionalMode && argIndex < numArgs) { + TINYFORMAT_ERROR("tinyformat: Not enough conversion specifiers in format string"); + } + break; + } bool spacePadPositive = false; int ntrunc = -1; - const char* fmtEnd = streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, - formatters, argIndex, numFormatters); - if (argIndex >= numFormatters) - { - // Check args remain after reading any variable width/precision - TINYFORMAT_ERROR("tinyformat: Not enough format arguments"); + const char* fmtEnd = streamStateFromFormat(out, positionalMode, spacePadPositive, ntrunc, fmt, + args, argIndex, numArgs); + // NB: argIndex may be incremented by reading variable width/precision + // in `streamStateFromFormat`, so do the bounds check here. + if (argIndex >= numArgs) { + TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); return; } - const FormatArg& arg = formatters[argIndex]; + const FormatArg& arg = args[argIndex]; // Format the arg into the stream. - if(!spacePadPositive) + if (!spacePadPositive) { arg.format(out, fmt, fmtEnd, ntrunc); - else - { + } + else { // The following is a special case with no direct correspondence // between stream formatting and the printf() behaviour. Simulate // it crudely by formatting into a temporary string stream and @@ -824,18 +916,17 @@ inline void formatImpl(std::ostream& out, const char* fmt, tmpStream.setf(std::ios::showpos); arg.format(tmpStream, fmt, fmtEnd, ntrunc); std::string result = tmpStream.str(); // allocates... yuck. - for(size_t i = 0, iend = result.size(); i < iend; ++i) - if(result[i] == '+') result[i] = ' '; + for (size_t i = 0, iend = result.size(); i < iend; ++i) { + if (result[i] == '+') + result[i] = ' '; + } out << result; } + if (!positionalMode) + ++argIndex; fmt = fmtEnd; } - // Print remaining part of format string. - fmt = printFormatStringLiteral(out, fmt); - if(*fmt != '\0') - TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); - // Restore stream state out.width(origWidth); out.precision(origPrecision); @@ -855,14 +946,14 @@ inline void formatImpl(std::ostream& out, const char* fmt, class FormatList { public: - FormatList(detail::FormatArg* formatters, int N) - : m_formatters(formatters), m_N(N) { } + FormatList(detail::FormatArg* args, int N) + : m_args(args), m_N(N) { } friend void vformat(std::ostream& out, const char* fmt, const FormatList& list); private: - const detail::FormatArg* m_formatters; + const detail::FormatArg* m_args; int m_N; }; @@ -879,29 +970,33 @@ class FormatListN : public FormatList public: #ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES template<typename... Args> - explicit FormatListN(const Args&... args) + FormatListN(const Args&... args) : FormatList(&m_formatterStore[0], N), m_formatterStore { FormatArg(args)... } { static_assert(sizeof...(args) == N, "Number of args must be N"); } #else // C++98 version void init(int) {} -# define TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR(n) \ - \ - template<TINYFORMAT_ARGTYPES(n)> \ - explicit FormatListN(TINYFORMAT_VARARGS(n)) \ - : FormatList(&m_formatterStore[0], n) \ - { assert(n == N); init(0, TINYFORMAT_PASSARGS(n)); } \ - \ - template<TINYFORMAT_ARGTYPES(n)> \ - void init(int i, TINYFORMAT_VARARGS(n)) \ - { \ - m_formatterStore[i] = FormatArg(v1); \ - init(i+1 TINYFORMAT_PASSARGS_TAIL(n)); \ +# define TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR(n) \ + \ + template<TINYFORMAT_ARGTYPES(n)> \ + FormatListN(TINYFORMAT_VARARGS(n)) \ + : FormatList(&m_formatterStore[0], n) \ + { TINYFORMAT_ASSERT(n == N); init(0, TINYFORMAT_PASSARGS(n)); } \ + \ + template<TINYFORMAT_ARGTYPES(n)> \ + void init(int i, TINYFORMAT_VARARGS(n)) \ + { \ + m_formatterStore[i] = FormatArg(v1); \ + init(i+1 TINYFORMAT_PASSARGS_TAIL(n)); \ } TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR) # undef TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR #endif + FormatListN(const FormatListN& other) + : FormatList(&m_formatterStore[0], N) + { std::copy(&other.m_formatterStore[0], &other.m_formatterStore[N], + &m_formatterStore[0]); } private: FormatArg m_formatterStore[N]; @@ -956,7 +1051,7 @@ TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_MAKEFORMATLIST) /// list of format arguments is held in a single function argument. inline void vformat(std::ostream& out, const char* fmt, FormatListRef list) { - detail::formatImpl(out, fmt, list.m_formatters, list.m_N); + detail::formatImpl(out, fmt, list.m_args, list.m_N); } @@ -993,6 +1088,7 @@ void printfln(const char* fmt, const Args&... args) std::cout << '\n'; } + #else // C++98 version inline void format(std::ostream& out, const char* fmt) @@ -1063,6 +1159,7 @@ std::string format(const std::string &fmt, const Args&... args) } // namespace tinyformat +// Added for Bitcoin Core: /** Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for details) */ #define strprintf tfm::format diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 3f40785c21..84118b36ef 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -501,7 +501,7 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe } return; } - service = LookupNumeric(std::string(service_id+".onion").c_str(), Params().GetDefaultPort()); + service = LookupNumeric(std::string(service_id+".onion"), Params().GetDefaultPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", GetPrivateKeyFile().string()); diff --git a/src/torcontrol.h b/src/torcontrol.h index e1a1a7937a..474a4d87d9 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txdb.cpp b/src/txdb.cpp index a7eb5f9f67..35bbdab00d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -336,7 +336,7 @@ public: vout.assign(vAvail.size(), CTxOut()); for (unsigned int i = 0; i < vAvail.size(); i++) { if (vAvail[i]) - ::Unserialize(s, CTxOutCompressor(vout[i])); + ::Unserialize(s, Using<TxOutCompression>(vout[i])); } // coinbase height ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); diff --git a/src/txdb.h b/src/txdb.h index 05bf4e4453..488c24f935 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,8 +20,6 @@ class CBlockIndex; class CCoinsViewDBCursor; class uint256; -//! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 10; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! -dbbatchsize default (bytes) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index e4c1fd4bc6..5768219f3a 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> +#include <optional.h> #include <validation.h> #include <policy/policy.h> #include <policy/fees.h> @@ -16,12 +17,13 @@ #include <util/system.h> #include <util/moneystr.h> #include <util/time.h> +#include <validationinterface.h> CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp) : tx(_tx), nFee(_nFee), nTxWeight(GetTransactionWeight(*tx)), nUsageSize(RecursiveDynamicUsage(tx)), nTime(_nTime), entryHeight(_entryHeight), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) + spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp), m_epoch(0) { nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); @@ -120,8 +122,6 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes // setMemPoolChildren will be updated, an assumption made in // UpdateForDescendants. for (const uint256 &hash : reverse_iterate(vHashesToUpdate)) { - // we cache the in-mempool children to avoid duplicate updates - setEntries setChildren; // calculate children from mapNextTx txiter it = mapTx.find(hash); if (it == mapTx.end()) { @@ -130,17 +130,21 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); // First calculate the children, and update setMemPoolChildren to // include them, and update their setMemPoolParents to include this tx. - for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { - const uint256 &childHash = iter->second->GetHash(); - txiter childIter = mapTx.find(childHash); - assert(childIter != mapTx.end()); - // We can skip updating entries we've encountered before or that - // are in the block (which are already accounted for). - if (setChildren.insert(childIter).second && !setAlreadyIncluded.count(childHash)) { - UpdateChild(it, childIter, true); - UpdateParent(childIter, it, true); + // we cache the in-mempool children to avoid duplicate updates + { + const auto epoch = GetFreshEpoch(); + for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { + const uint256 &childHash = iter->second->GetHash(); + txiter childIter = mapTx.find(childHash); + assert(childIter != mapTx.end()); + // We can skip updating entries we've encountered before or that + // are in the block (which are already accounted for). + if (!visited(childIter) && !setAlreadyIncluded.count(childHash)) { + UpdateChild(it, childIter, true); + UpdateParent(childIter, it, true); + } } - } + } // release epoch guard for UpdateForDescendants UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded); } } @@ -155,7 +159,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr // GetMemPoolParents() is only valid for entries in the mempool, so we // iterate mapTx to find parents. for (unsigned int i = 0; i < tx.vin.size(); i++) { - boost::optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); + Optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); if (piter) { parentHashes.insert(*piter); if (parentHashes.size() + 1 > limitAncestorCount) { @@ -323,7 +327,7 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, } CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) - : nTransactionsUpdated(0), minerPolicyEstimator(estimator) + : nTransactionsUpdated(0), minerPolicyEstimator(estimator), m_epoch(0), m_has_epoch_guard(false) { _clear(); //lock free clear @@ -402,7 +406,12 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) { - NotifyEntryRemoved(it->GetSharedTx(), reason); + CTransactionRef ptx = it->GetSharedTx(); + NotifyEntryRemoved(ptx, reason); + if (reason != MemPoolRemovalReason::BLOCK && reason != MemPoolRemovalReason::CONFLICT) { + GetMainSignals().TransactionRemovedFromMempool(ptx); + } + const uint256 hash = it->GetTx().GetHash(); for (const CTxIn& txin : it->GetTx().vin) mapNextTx.erase(txin.prevout); @@ -591,9 +600,9 @@ void CTxMemPool::clear() static void CheckInputsAndUpdateCoins(const CTransaction& tx, CCoinsViewCache& mempoolDuplicate, const int64_t spendheight) { - CValidationState state; + TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass CAmount txfee = 0; - bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, state, mempoolDuplicate, spendheight, txfee); + bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, std::numeric_limits<int>::max()); } @@ -860,11 +869,11 @@ const CTransaction* CTxMemPool::GetConflictTx(const COutPoint& prevout) const return it == mapNextTx.end() ? nullptr : it->second; } -boost::optional<CTxMemPool::txiter> CTxMemPool::GetIter(const uint256& txid) const +Optional<CTxMemPool::txiter> CTxMemPool::GetIter(const uint256& txid) const { auto it = mapTx.find(txid); if (it != mapTx.end()) return it; - return boost::optional<txiter>{}; + return Optional<txiter>{}; } CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set<uint256>& hashes) const @@ -1098,4 +1107,23 @@ void CTxMemPool::SetIsLoaded(bool loaded) m_is_loaded = loaded; } + +CTxMemPool::EpochGuard CTxMemPool::GetFreshEpoch() const +{ + return EpochGuard(*this); +} +CTxMemPool::EpochGuard::EpochGuard(const CTxMemPool& in) : pool(in) +{ + assert(!pool.m_has_epoch_guard); + ++pool.m_epoch; + pool.m_has_epoch_guard = true; +} + +CTxMemPool::EpochGuard::~EpochGuard() +{ + // prevents stale results being used + ++pool.m_epoch; + pool.m_has_epoch_guard = false; +} + SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index b51e800001..de11d626b4 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,6 +17,7 @@ #include <coins.h> #include <crypto/siphash.h> #include <indirectmap.h> +#include <optional.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <sync.h> @@ -29,7 +30,7 @@ #include <boost/signals2/signal.hpp> class CBlockIndex; -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -49,8 +50,6 @@ struct LockPoints LockPoints() : height(0), time(0), maxInputBlock(nullptr) { } }; -class CTxMemPool; - /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well @@ -130,6 +129,7 @@ public: int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable uint64_t m_epoch; //!< epoch when last touched, useful for graph algorithms }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. @@ -454,6 +454,8 @@ private: mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; mutable double rollingMinimumFeeRate; //!< minimum fee to get into the pool, decreases exponentially + mutable uint64_t m_epoch; + mutable bool m_has_epoch_guard; void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -602,7 +604,7 @@ public: const CTransaction* GetConflictTx(const COutPoint& prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Returns an iterator to the given hash, if found */ - boost::optional<txiter> GetIter(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); + Optional<txiter> GetIter(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Translate a set of hashes into a set of pool iterators to avoid repeated lookups */ setEntries GetIterSet(const std::set<uint256>& hashes) const EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -737,6 +739,55 @@ private: * removal. */ void removeUnchecked(txiter entry, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); +public: + /** EpochGuard: RAII-style guard for using epoch-based graph traversal algorithms. + * When walking ancestors or descendants, we generally want to avoid + * visiting the same transactions twice. Some traversal algorithms use + * std::set (or setEntries) to deduplicate the transaction we visit. + * However, use of std::set is algorithmically undesirable because it both + * adds an asymptotic factor of O(log n) to traverals cost and triggers O(n) + * more dynamic memory allocations. + * In many algorithms we can replace std::set with an internal mempool + * counter to track the time (or, "epoch") that we began a traversal, and + * check + update a per-transaction epoch for each transaction we look at to + * determine if that transaction has not yet been visited during the current + * traversal's epoch. + * Algorithms using std::set can be replaced on a one by one basis. + * Both techniques are not fundamentally incomaptible across the codebase. + * Generally speaking, however, the remaining use of std::set for mempool + * traversal should be viewed as a TODO for replacement with an epoch based + * traversal, rather than a preference for std::set over epochs in that + * algorithm. + */ + class EpochGuard { + const CTxMemPool& pool; + public: + EpochGuard(const CTxMemPool& in); + ~EpochGuard(); + }; + // N.B. GetFreshEpoch modifies mutable state via the EpochGuard construction + // (and later destruction) + EpochGuard GetFreshEpoch() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + /** visited marks a CTxMemPoolEntry as having been traversed + * during the lifetime of the most recently created EpochGuard + * and returns false if we are the first visitor, true otherwise. + * + * An EpochGuard must be held when visited is called or an assert will be + * triggered. + * + */ + bool visited(txiter it) const EXCLUSIVE_LOCKS_REQUIRED(cs) { + assert(m_has_epoch_guard); + bool ret = it->m_epoch >= m_epoch; + it->m_epoch = std::max(it->m_epoch, m_epoch); + return ret; + } + + bool visited(Optional<txiter> it) const EXCLUSIVE_LOCKS_REQUIRED(cs) { + assert(m_has_epoch_guard); + return !it || visited(*it); + } }; /** diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index d310637145..85bb746d19 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2018 The Bitcoin Core developers +// Copyright (c) 2010-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,7 +16,6 @@ struct UISignals { boost::signals2::signal<CClientUIInterface::NotifyNumConnectionsChangedSig> NotifyNumConnectionsChanged; boost::signals2::signal<CClientUIInterface::NotifyNetworkActiveChangedSig> NotifyNetworkActiveChanged; boost::signals2::signal<CClientUIInterface::NotifyAlertChangedSig> NotifyAlertChanged; - boost::signals2::signal<CClientUIInterface::LoadWalletSig> LoadWallet; boost::signals2::signal<CClientUIInterface::ShowProgressSig> ShowProgress; boost::signals2::signal<CClientUIInterface::NotifyBlockTipSig> NotifyBlockTip; boost::signals2::signal<CClientUIInterface::NotifyHeaderTipSig> NotifyHeaderTip; @@ -36,7 +35,6 @@ ADD_SIGNALS_IMPL_WRAPPER(InitMessage); ADD_SIGNALS_IMPL_WRAPPER(NotifyNumConnectionsChanged); ADD_SIGNALS_IMPL_WRAPPER(NotifyNetworkActiveChanged); ADD_SIGNALS_IMPL_WRAPPER(NotifyAlertChanged); -ADD_SIGNALS_IMPL_WRAPPER(LoadWallet); ADD_SIGNALS_IMPL_WRAPPER(ShowProgress); ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); @@ -48,7 +46,6 @@ void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_s void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } -void CClientUIInterface::LoadWallet(std::unique_ptr<interfaces::Wallet>& wallet) { return g_ui_signals.LoadWallet(wallet); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } diff --git a/src/ui_interface.h b/src/ui_interface.h index 9efc2db391..b402177b85 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,10 +17,6 @@ class connection; } } // namespace boost -namespace interfaces { -class Wallet; -} // namespace interfaces - /** General change type (added, updated, removed). */ enum ChangeType { @@ -105,9 +101,6 @@ public: */ ADD_SIGNALS_DECL_WRAPPER(NotifyAlertChanged, void, ); - /** A wallet has been loaded. */ - ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::unique_ptr<interfaces::Wallet>& wallet); - /** * Show progress e.g. for verifychain. * resume_possible indicates shutting down now will result in the current progress action resuming upon restart. diff --git a/src/uint256.cpp b/src/uint256.cpp index ee597e1877..a943e71062 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,3 +75,8 @@ template std::string base_blob<256>::GetHex() const; template std::string base_blob<256>::ToString() const; template void base_blob<256>::SetHex(const char*); template void base_blob<256>::SetHex(const std::string&); + +uint256& UINT256_ONE() { + static uint256* one = new uint256(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); + return *one; +} diff --git a/src/uint256.h b/src/uint256.h index 60c5e0554c..b36598f572 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -144,4 +144,6 @@ inline uint256 uint256S(const std::string& str) return rv; } +uint256& UINT256_ONE(); + #endif // BITCOIN_UINT256_H diff --git a/src/undo.h b/src/undo.h index 3f50f4caad..2009c721ab 100644 --- a/src/undo.h +++ b/src/undo.h @@ -32,7 +32,7 @@ public: // Required to maintain compatibility with older undo format. ::Serialize(s, (unsigned char)0); } - ::Serialize(s, CTxOutCompressor(REF(txout->out))); + ::Serialize(s, Using<TxOutCompression>(REF(txout->out))); } explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {} @@ -56,7 +56,7 @@ public: unsigned int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, CTxOutCompressor(REF(txout->out))); + ::Unserialize(s, Using<TxOutCompression>(REF(txout->out))); } explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {} diff --git a/src/univalue/README.md b/src/univalue/README.md index 36aa786a4c..7c62c33970 100644 --- a/src/univalue/README.md +++ b/src/univalue/README.md @@ -12,21 +12,10 @@ an arbitrary depth. This class is aligned with the JSON standard, [RFC 7159](https://tools.ietf.org/html/rfc7159.html). -## Installation +## Library usage -This project is a standard GNU -[autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html) -project. Build and install instructions are available in the `INSTALL` -file provided with GNU autotools. - -``` -$ ./autogen.sh -$ ./configure -$ make -``` - -## Design - -UniValue provides a single dynamic RAII C++ object class, -and minimizes template use (contra json_spirit). +This is a fork of univalue used by Bitcoin Core. It is not maintained for usage +by other projects. Notably, the API may break in non-backward-compatible ways. +Other projects looking for a maintained library should use the upstream +univalue at https://github.com/jgarzik/univalue. diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 91b104e56e..6080516353 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -47,7 +47,6 @@ public: std::string s(val_); setStr(s); } - ~UniValue() {} void clear(); diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index eabcf2dad1..0ad6146545 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -35,7 +35,7 @@ bool ParseInt32(const std::string& str, int32_t *out) errno = 0; // strtol will not set errno if valid long int n = strtol(str.c_str(), &endp, 10); if(out) *out = (int32_t)n; - // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow + // Note that strtol returns a *long int*, so even if strtol doesn't report an over/underflow // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit // platforms the size of these types may be different. return endp && *endp == 0 && !errno && diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp new file mode 100644 index 0000000000..ac230e9ee5 --- /dev/null +++ b/src/util/asmap.cpp @@ -0,0 +1,97 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <vector> +#include <assert.h> +#include <crypto/common.h> + +namespace { + +uint32_t DecodeBits(std::vector<bool>::const_iterator& bitpos, uint8_t minval, const std::vector<uint8_t> &bit_sizes) +{ + uint32_t val = minval; + bool bit; + for (std::vector<uint8_t>::const_iterator bit_sizes_it = bit_sizes.begin(); + bit_sizes_it != bit_sizes.end(); ++bit_sizes_it) { + if (bit_sizes_it + 1 != bit_sizes.end()) { + bit = *bitpos; + bitpos++; + } else { + bit = 0; + } + if (bit) { + val += (1 << *bit_sizes_it); + } else { + for (int b = 0; b < *bit_sizes_it; b++) { + bit = *bitpos; + bitpos++; + val += bit << (*bit_sizes_it - 1 - b); + } + return val; + } + } + return -1; +} + +const std::vector<uint8_t> TYPE_BIT_SIZES{0, 0, 1}; +uint32_t DecodeType(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 0, TYPE_BIT_SIZES); +} + +const std::vector<uint8_t> ASN_BIT_SIZES{15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; +uint32_t DecodeASN(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 1, ASN_BIT_SIZES); +} + + +const std::vector<uint8_t> MATCH_BIT_SIZES{1, 2, 3, 4, 5, 6, 7, 8}; +uint32_t DecodeMatch(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 2, MATCH_BIT_SIZES); +} + + +const std::vector<uint8_t> JUMP_BIT_SIZES{5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; +uint32_t DecodeJump(std::vector<bool>::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 17, JUMP_BIT_SIZES); +} + +} + +uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip) +{ + std::vector<bool>::const_iterator pos = asmap.begin(); + uint8_t bits = ip.size(); + uint8_t default_asn = 0; + uint32_t opcode, jump, match, matchlen; + while (1) { + assert(pos != asmap.end()); + opcode = DecodeType(pos); + if (opcode == 0) { + return DecodeASN(pos); + } else if (opcode == 1) { + jump = DecodeJump(pos); + if (ip[ip.size() - bits]) { + pos += jump; + } + bits--; + } else if (opcode == 2) { + match = DecodeMatch(pos); + matchlen = CountBits(match) - 1; + for (uint32_t bit = 0; bit < matchlen; bit++) { + if ((ip[ip.size() - bits]) != ((match >> (matchlen - 1 - bit)) & 1)) { + return default_asn; + } + bits--; + } + } else if (opcode == 3) { + default_asn = DecodeASN(pos); + } else { + assert(0); + } + } +} diff --git a/src/util/asmap.h b/src/util/asmap.h new file mode 100644 index 0000000000..a0e14013c5 --- /dev/null +++ b/src/util/asmap.h @@ -0,0 +1,10 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ASMAP_H +#define BITCOIN_UTIL_ASMAP_H + +uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip); + +#endif // BITCOIN_UTIL_ASMAP_H diff --git a/src/util/check.h b/src/util/check.h new file mode 100644 index 0000000000..d18887ae95 --- /dev/null +++ b/src/util/check.h @@ -0,0 +1,41 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_CHECK_H +#define BITCOIN_UTIL_CHECK_H + +#include <tinyformat.h> + +#include <stdexcept> + +class NonFatalCheckError : public std::runtime_error +{ + using std::runtime_error::runtime_error; +}; + +/** + * Throw a NonFatalCheckError when the condition evaluates to false + * + * This should only be used + * - where the condition is assumed to be true, not for error handling or validating user input + * - where a failure to fulfill the condition is recoverable and does not abort the program + * + * For example in RPC code, where it is undersirable to crash the whole program, this can be generally used to replace + * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC + * caller, which can then report the issue to the developers. + */ +#define CHECK_NONFATAL(condition) \ + do { \ + if (!(condition)) { \ + throw NonFatalCheckError( \ + strprintf("%s:%d (%s)\n" \ + "Internal bug detected: '%s'\n" \ + "You may report this issue here: %s\n", \ + __FILE__, __LINE__, __func__, \ + (#condition), \ + PACKAGE_BUGREPORT)); \ + } \ + } while (false) + +#endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/error.cpp b/src/util/error.cpp index aa44ed3e3a..72a6e87cde 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2018 The Bitcoin Core developers +// Copyright (c) 2010-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/error.h b/src/util/error.h index f540b0020d..61af88ddea 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2018 The Bitcoin Core developers +// Copyright (c) 2010-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/fees.cpp b/src/util/fees.cpp index cf16d5e44f..b335bfa666 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -1,10 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <util/fees.h> + #include <policy/fees.h> +#include <map> #include <string> std::string StringForFeeReason(FeeReason reason) { diff --git a/src/util/fees.h b/src/util/fees.h index fc355ce9c2..a930c8935a 100644 --- a/src/util/fees.h +++ b/src/util/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_UTIL_FEES_H diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index ba5a12e58c..2797f450ca 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #include <tinyformat.h> #include <util/strencodings.h> +#include <util/string.h> std::string FormatMoney(const CAmount& n) { @@ -32,6 +33,9 @@ std::string FormatMoney(const CAmount& n) bool ParseMoney(const std::string& str, CAmount& nRet) { + if (!ValidAsCString(str)) { + return false; + } return ParseMoney(str.c_str(), nRet); } diff --git a/src/util/moneystr.h b/src/util/moneystr.h index 4d0218911a..027c7e2e53 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/rbf.cpp b/src/util/rbf.cpp index d520a9606d..ef536bdcad 100644 --- a/src/util/rbf.cpp +++ b/src/util/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/rbf.h b/src/util/rbf.h index d3ef110628..6a20b37de5 100644 --- a/src/util/rbf.h +++ b/src/util/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/settings.cpp b/src/util/settings.cpp new file mode 100644 index 0000000000..e4979df755 --- /dev/null +++ b/src/util/settings.cpp @@ -0,0 +1,178 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/settings.h> + +#include <univalue.h> + +namespace util { +namespace { + +enum class Source { + FORCED, + COMMAND_LINE, + CONFIG_FILE_NETWORK_SECTION, + CONFIG_FILE_DEFAULT_SECTION +}; + +//! Merge settings from multiple sources in precedence order: +//! Forced config > command line > config file network-specific section > config file default section +//! +//! This function is provided with a callback function fn that contains +//! specific logic for how to merge the sources. +template <typename Fn> +static void MergeSettings(const Settings& settings, const std::string& section, const std::string& name, Fn&& fn) +{ + // Merge in the forced settings + if (auto* value = FindKey(settings.forced_settings, name)) { + fn(SettingsSpan(*value), Source::FORCED); + } + // Merge in the command-line options + if (auto* values = FindKey(settings.command_line_options, name)) { + fn(SettingsSpan(*values), Source::COMMAND_LINE); + } + // Merge in the network-specific section of the config file + if (!section.empty()) { + if (auto* map = FindKey(settings.ro_config, section)) { + if (auto* values = FindKey(*map, name)) { + fn(SettingsSpan(*values), Source::CONFIG_FILE_NETWORK_SECTION); + } + } + } + // Merge in the default section of the config file + if (auto* map = FindKey(settings.ro_config, "")) { + if (auto* values = FindKey(*map, name)) { + fn(SettingsSpan(*values), Source::CONFIG_FILE_DEFAULT_SECTION); + } + } +} +} // namespace + +SettingsValue GetSetting(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config, + bool get_chain_name) +{ + SettingsValue result; + bool done = false; // Done merging any more settings sources. + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + // Weird behavior preserved for backwards compatibility: Apply negated + // setting even if non-negated setting would be ignored. A negated + // value in the default section is applied to network specific options, + // even though normal non-negated values there would be ignored. + const bool never_ignore_negated_setting = span.last_negated(); + + // Weird behavior preserved for backwards compatibility: Take first + // assigned value instead of last. In general, later settings take + // precedence over early settings, but for backwards compatibility in + // the config file the precedence is reversed for all settings except + // chain name settings. + const bool reverse_precedence = + (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && + !get_chain_name; + + // Weird behavior preserved for backwards compatibility: Negated + // -regtest and -testnet arguments which you would expect to override + // values set in the configuration file are currently accepted but + // silently ignored. It would be better to apply these just like other + // negated values, or at least warn they are ignored. + const bool skip_negated_command_line = get_chain_name; + + if (done) return; + + // Ignore settings in default config section if requested. + if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION && + !never_ignore_negated_setting) { + return; + } + + // Skip negated command line settings. + if (skip_negated_command_line && span.last_negated()) return; + + if (!span.empty()) { + result = reverse_precedence ? span.begin()[0] : span.end()[-1]; + done = true; + } else if (span.last_negated()) { + result = false; + done = true; + } + }); + return result; +} + +std::vector<SettingsValue> GetSettingsList(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config) +{ + std::vector<SettingsValue> result; + bool done = false; // Done merging any more settings sources. + bool prev_negated_empty = false; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + // Weird behavior preserved for backwards compatibility: Apply config + // file settings even if negated on command line. Negating a setting on + // command line will ignore earlier settings on the command line and + // ignore settings in the config file, unless the negated command line + // value is followed by non-negated value, in which case config file + // settings will be brought back from the dead (but earlier command + // line settings will still be ignored). + const bool add_zombie_config_values = + (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && + !prev_negated_empty; + + // Ignore settings in default config section if requested. + if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION) return; + + // Add new settings to the result if isn't already complete, or if the + // values are zombies. + if (!done || add_zombie_config_values) { + for (const auto& value : span) { + if (value.isArray()) { + result.insert(result.end(), value.getValues().begin(), value.getValues().end()); + } else { + result.push_back(value); + } + } + } + + // If a setting was negated, or if a setting was forced, set + // done to true to ignore any later lower priority settings. + done |= span.negated() > 0 || source == Source::FORCED; + + // Update the negated and empty state used for the zombie values check. + prev_negated_empty |= span.last_negated() && result.empty(); + }); + return result; +} + +bool OnlyHasDefaultSectionSetting(const Settings& settings, const std::string& section, const std::string& name) +{ + bool has_default_section_setting = false; + bool has_other_setting = false; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + if (span.empty()) return; + else if (source == Source::CONFIG_FILE_DEFAULT_SECTION) has_default_section_setting = true; + else has_other_setting = true; + }); + // If a value is set in the default section and not explicitly overwritten by the + // user on the command line or in a different section, then we want to enable + // warnings about the value being ignored. + return has_default_section_setting && !has_other_setting; +} + +SettingsSpan::SettingsSpan(const std::vector<SettingsValue>& vec) noexcept : SettingsSpan(vec.data(), vec.size()) {} +const SettingsValue* SettingsSpan::begin() const { return data + negated(); } +const SettingsValue* SettingsSpan::end() const { return data + size; } +bool SettingsSpan::empty() const { return size == 0 || last_negated(); } +bool SettingsSpan::last_negated() const { return size > 0 && data[size - 1].isFalse(); } +size_t SettingsSpan::negated() const +{ + for (size_t i = size; i > 0; --i) { + if (data[i - 1].isFalse()) return i; // Return number of negated values (position of last false value) + } + return 0; +} + +} // namespace util diff --git a/src/util/settings.h b/src/util/settings.h new file mode 100644 index 0000000000..1d03639fa2 --- /dev/null +++ b/src/util/settings.h @@ -0,0 +1,94 @@ +// Copyright (c) 2019-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SETTINGS_H +#define BITCOIN_UTIL_SETTINGS_H + +#include <map> +#include <string> +#include <vector> + +class UniValue; + +namespace util { + +//! Settings value type (string/integer/boolean/null variant). +//! +//! @note UniValue is used here for convenience and because it can be easily +//! serialized in a readable format. But any other variant type that can +//! be assigned strings, int64_t, and bool values and has get_str(), +//! get_int64(), get_bool(), isNum(), isBool(), isFalse(), isTrue() and +//! isNull() methods can be substituted if there's a need to move away +//! from UniValue. (An implementation with boost::variant was posted at +//! https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812) +using SettingsValue = UniValue; + +//! Stored bitcoin settings. This struct combines settings from the command line +//! and a read-only configuration file. +struct Settings { + //! Map of setting name to forced setting value. + std::map<std::string, SettingsValue> forced_settings; + //! Map of setting name to list of command line values. + std::map<std::string, std::vector<SettingsValue>> command_line_options; + //! Map of config section name and setting name to list of config file values. + std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config; +}; + +//! Get settings value from combined sources: forced settings, command line +//! arguments and the read-only config file. +//! +//! @param ignore_default_section_config - ignore values in the default section +//! of the config file (part before any +//! [section] keywords) +//! @param get_chain_name - enable special backwards compatible behavior +//! for GetChainName +SettingsValue GetSetting(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config, + bool get_chain_name); + +//! Get combined setting value similar to GetSetting(), except if setting was +//! specified multiple times, return a list of all the values specified. +std::vector<SettingsValue> GetSettingsList(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config); + +//! Return true if a setting is set in the default config file section, and not +//! overridden by a higher priority command-line or network section value. +//! +//! This is used to provide user warnings about values that might be getting +//! ignored unintentionally. +bool OnlyHasDefaultSectionSetting(const Settings& settings, const std::string& section, const std::string& name); + +//! Accessor for list of settings that skips negated values when iterated over. +//! The last boolean `false` value in the list and all earlier values are +//! considered negated. +struct SettingsSpan { + explicit SettingsSpan() = default; + explicit SettingsSpan(const SettingsValue& value) noexcept : SettingsSpan(&value, 1) {} + explicit SettingsSpan(const SettingsValue* data, size_t size) noexcept : data(data), size(size) {} + explicit SettingsSpan(const std::vector<SettingsValue>& vec) noexcept; + const SettingsValue* begin() const; //!< Pointer to first non-negated value. + const SettingsValue* end() const; //!< Pointer to end of values. + bool empty() const; //!< True if there are any non-negated values. + bool last_negated() const; //!< True if the last value is negated. + size_t negated() const; //!< Number of negated values. + + const SettingsValue* data = nullptr; + size_t size = 0; +}; + +//! Map lookup helper. +template <typename Map, typename Key> +auto FindKey(Map&& map, Key&& key) -> decltype(&map.at(key)) +{ + auto it = map.find(key); + return it == map.end() ? nullptr : &it->second; +} + +} // namespace util + +#endif // BITCOIN_UTIL_SETTINGS_H diff --git a/src/util/spanparsing.cpp b/src/util/spanparsing.cpp index 0c8575399a..0f68254f2c 100644 --- a/src/util/spanparsing.cpp +++ b/src/util/spanparsing.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/spanparsing.h b/src/util/spanparsing.h index 63f54758bd..fa2e698e6d 100644 --- a/src/util/spanparsing.h +++ b/src/util/spanparsing.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 1e7d24c71c..eec1a52e95 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -1,9 +1,10 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> +#include <util/string.h> #include <tinyformat.h> @@ -138,7 +139,7 @@ std::string EncodeBase64(const unsigned char* pch, size_t len) std::string EncodeBase64(const std::string& str) { - return EncodeBase64((const unsigned char*)str.c_str(), str.size()); + return EncodeBase64((const unsigned char*)str.data(), str.size()); } std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) @@ -190,6 +191,12 @@ std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) std::string DecodeBase64(const std::string& str, bool* pf_invalid) { + if (!ValidAsCString(str)) { + if (pf_invalid) { + *pf_invalid = true; + } + return {}; + } std::vector<unsigned char> vchRet = DecodeBase64(str.c_str(), pf_invalid); return std::string((const char*)vchRet.data(), vchRet.size()); } @@ -207,7 +214,7 @@ std::string EncodeBase32(const unsigned char* pch, size_t len) std::string EncodeBase32(const std::string& str) { - return EncodeBase32((const unsigned char*)str.c_str(), str.size()); + return EncodeBase32((const unsigned char*)str.data(), str.size()); } std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) @@ -259,6 +266,12 @@ std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) std::string DecodeBase32(const std::string& str, bool* pf_invalid) { + if (!ValidAsCString(str)) { + if (pf_invalid) { + *pf_invalid = true; + } + return {}; + } std::vector<unsigned char> vchRet = DecodeBase32(str.c_str(), pf_invalid); return std::string((const char*)vchRet.data(), vchRet.size()); } @@ -269,7 +282,7 @@ NODISCARD static bool ParsePrechecks(const std::string& str) return false; if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size()-1]))) // No padding allowed return false; - if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed + if (!ValidAsCString(str)) // No embedded NUL characters allowed return false; return true; } diff --git a/src/util/strencodings.h b/src/util/strencodings.h index e35b2ab857..ccc4edac12 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/string.h b/src/util/string.h index 76a83a4949..3db8fc8b98 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -5,9 +5,22 @@ #ifndef BITCOIN_UTIL_STRING_H #define BITCOIN_UTIL_STRING_H +#include <attributes.h> + +#include <cstring> #include <string> #include <vector> +NODISCARD inline std::string TrimString(const std::string& str, const std::string& pattern = " \f\n\r\t\v") +{ + std::string::size_type front = str.find_first_not_of(pattern); + if (front == std::string::npos) { + return std::string(); + } + std::string::size_type end = str.find_last_not_of(pattern); + return str.substr(front, end - front + 1); +} + /** * Join a list of items * @@ -31,4 +44,12 @@ inline std::string Join(const std::vector<std::string>& list, const std::string& return Join(list, separator, [](const std::string& i) { return i; }); } +/** + * Check if a string does not contain any embedded NUL (\0) characters + */ +NODISCARD inline bool ValidAsCString(const std::string& str) noexcept +{ + return str.size() == strlen(str.c_str()); +} + #endif // BITCOIN_UTIL_STRENCODINGS_H diff --git a/src/util/system.cpp b/src/util/system.cpp index 526bf559c3..ff3967c577 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #include <chainparamsbase.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/translation.h> @@ -63,6 +64,8 @@ #endif #include <thread> +#include <typeinfo> +#include <univalue.h> // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); @@ -161,105 +164,10 @@ static bool InterpretBool(const std::string& strValue) return (atoi(strValue) != 0); } -/** Internal helper functions for ArgsManager */ -class ArgsManagerHelper { -public: - typedef std::map<std::string, std::vector<std::string>> MapArgs; - - /** Determine whether to use config settings in the default section, - * See also comments around ArgsManager::ArgsManager() below. */ - static inline bool UseDefaultSection(const ArgsManager& am, const std::string& arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) - { - return (am.m_network == CBaseChainParams::MAIN || am.m_network_only_args.count(arg) == 0); - } - - /** Convert regular argument into the network-specific setting */ - static inline std::string NetworkArg(const ArgsManager& am, const std::string& arg) - { - assert(arg.length() > 1 && arg[0] == '-'); - return "-" + am.m_network + "." + arg.substr(1); - } - - /** Find arguments in a map and add them to a vector */ - static inline void AddArgs(std::vector<std::string>& res, const MapArgs& map_args, const std::string& arg) - { - auto it = map_args.find(arg); - if (it != map_args.end()) { - res.insert(res.end(), it->second.begin(), it->second.end()); - } - } - - /** Return true/false if an argument is set in a map, and also - * return the first (or last) of the possibly multiple values it has - */ - static inline std::pair<bool,std::string> GetArgHelper(const MapArgs& map_args, const std::string& arg, bool getLast = false) - { - auto it = map_args.find(arg); - - if (it == map_args.end() || it->second.empty()) { - return std::make_pair(false, std::string()); - } - - if (getLast) { - return std::make_pair(true, it->second.back()); - } else { - return std::make_pair(true, it->second.front()); - } - } - - /* Get the string value of an argument, returning a pair of a boolean - * indicating the argument was found, and the value for the argument - * if it was found (or the empty string if not found). - */ - static inline std::pair<bool,std::string> GetArg(const ArgsManager &am, const std::string& arg) - { - LOCK(am.cs_args); - std::pair<bool,std::string> found_result(false, std::string()); - - // We pass "true" to GetArgHelper in order to return the last - // argument value seen from the command line (so "bitcoind -foo=bar - // -foo=baz" gives GetArg(am,"foo")=={true,"baz"} - found_result = GetArgHelper(am.m_override_args, arg, true); - if (found_result.first) { - return found_result; - } - - // But in contrast we return the first argument seen in a config file, - // so "foo=bar \n foo=baz" in the config file gives - // GetArg(am,"foo")={true,"bar"} - if (!am.m_network.empty()) { - found_result = GetArgHelper(am.m_config_args, NetworkArg(am, arg)); - if (found_result.first) { - return found_result; - } - } - - if (UseDefaultSection(am, arg)) { - found_result = GetArgHelper(am.m_config_args, arg); - if (found_result.first) { - return found_result; - } - } - - return found_result; - } - - /* Special test for -testnet and -regtest args, because we - * don't want to be confused by craziness like "[regtest] testnet=1" - */ - static inline bool GetNetBoolArg(const ArgsManager &am, const std::string& net_arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) - { - std::pair<bool,std::string> found_result(false,std::string()); - found_result = GetArgHelper(am.m_override_args, net_arg, true); - if (!found_result.first) { - found_result = GetArgHelper(am.m_config_args, net_arg, true); - if (!found_result.first) { - return false; // not set - } - } - return InterpretBool(found_result.second); // is set, so evaluate - } -}; +static std::string SettingName(const std::string& arg) +{ + return arg.size() > 0 && arg[0] == '-' ? arg.substr(1) : arg; +} /** * Interpret -nofoo as if the user supplied -foo=0. @@ -268,13 +176,12 @@ public: * checks whether there was a double-negative (-nofoo=0 -> -foo=1). * * If there was not a double negative, it removes the "no" from the key - * and clears the args vector to indicate a negated option. + * and returns false. * - * If there was a double negative, it removes "no" from the key, sets the - * value to "1" and pushes the key and the updated value to the args vector. + * If there was a double negative, it removes "no" from the key, and + * returns true. * - * If there was no "no", it leaves key and value untouched and pushes them - * to the args vector. + * If there was no "no", it returns the string value untouched. * * Where an option was negated can be later checked using the * IsArgNegated() method. One use case for this is to have a way to disable @@ -282,34 +189,39 @@ public: * that debug log output is not sent to any file at all). */ -NODISCARD static bool InterpretOption(std::string key, std::string val, unsigned int flags, - std::map<std::string, std::vector<std::string>>& args, - std::string& error) +static util::SettingsValue InterpretOption(std::string& section, std::string& key, const std::string& value) { - assert(key[0] == '-'); - + // Split section name from key name for keys like "testnet.foo" or "regtest.bar" size_t option_index = key.find('.'); - if (option_index == std::string::npos) { - option_index = 1; - } else { - ++option_index; + if (option_index != std::string::npos) { + section = key.substr(0, option_index); + key.erase(0, option_index + 1); } - if (key.substr(option_index, 2) == "no") { - key.erase(option_index, 2); - if (flags & ArgsManager::ALLOW_BOOL) { - if (InterpretBool(val)) { - args[key].clear(); - return true; - } - // Double negatives like -nofoo=0 are supported (but discouraged) - LogPrintf("Warning: parsed potentially confusing double-negative %s=%s\n", key, val); - val = "1"; - } else { - error = strprintf("Negating of %s is meaningless and therefore forbidden", key.c_str()); - return false; + if (key.substr(0, 2) == "no") { + key.erase(0, 2); + // Double negatives like -nofoo=0 are supported (but discouraged) + if (!InterpretBool(value)) { + LogPrintf("Warning: parsed potentially confusing double-negative -%s=%s\n", key, value); + return true; } + return false; + } + return value; +} + +/** + * Check settings value validity according to flags. + * + * TODO: Add more meaningful error checks here in the future + * See "here's how the flags are meant to behave" in + * https://github.com/bitcoin/bitcoin/pull/16097#issuecomment-514627823 + */ +static bool CheckValid(const std::string& key, const util::SettingsValue& val, unsigned int flags, std::string& error) +{ + if (val.isBool() && !(flags & ArgsManager::ALLOW_BOOL)) { + error = strprintf("Negating of -%s is meaningless and therefore forbidden", key); + return false; } - args[key].push_back(val); return true; } @@ -331,22 +243,9 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const if (m_network == CBaseChainParams::MAIN) return std::set<std::string> {}; for (const auto& arg : m_network_only_args) { - std::pair<bool, std::string> found_result; - - // if this option is overridden it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_override_args, arg); - if (found_result.first) continue; - - // if there's a network-specific value for this option, it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_config_args, ArgsManagerHelper::NetworkArg(*this, arg)); - if (found_result.first) continue; - - // if there isn't a default value for this option, it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_config_args, arg); - if (!found_result.first) continue; - - // otherwise, issue a warning - unsuitables.insert(arg); + if (OnlyHasDefaultSectionSetting(m_settings, m_network, SettingName(arg))) { + unsuitables.insert(arg); + } } return unsuitables; } @@ -375,7 +274,7 @@ void ArgsManager::SelectConfigNetwork(const std::string& network) bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::string& error) { LOCK(cs_args); - m_override_args.clear(); + m_settings.command_line_options.clear(); for (int i = 1; i < argc; i++) { std::string key(argv[i]); @@ -408,121 +307,86 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin if (key.length() > 1 && key[1] == '-') key.erase(0, 1); - const unsigned int flags = FlagsOfKnownArg(key); + // Transform -foo to foo + key.erase(0, 1); + std::string section; + util::SettingsValue value = InterpretOption(section, key, val); + Optional<unsigned int> flags = GetArgFlags('-' + key); if (flags) { - if (!InterpretOption(key, val, flags, m_override_args, error)) { + if (!CheckValid(key, value, *flags, error)) { return false; } + // Weird behavior preserved for backwards compatibility: command + // line options with section prefixes are allowed but ignored. It + // would be better if these options triggered the Invalid parameter + // error below. + if (section.empty()) { + m_settings.command_line_options[key].push_back(value); + } } else { - error = strprintf("Invalid parameter %s", key.c_str()); + error = strprintf("Invalid parameter -%s", key); return false; } } - // we do not allow -includeconf from command line, so we clear it here - auto it = m_override_args.find("-includeconf"); - if (it != m_override_args.end()) { - if (it->second.size() > 0) { - for (const auto& ic : it->second) { - error += "-includeconf cannot be used from commandline; -includeconf=" + ic + "\n"; - } - return false; + // we do not allow -includeconf from command line + bool success = true; + if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + for (const auto& include : util::SettingsSpan(*includes)) { + error += "-includeconf cannot be used from commandline; -includeconf=" + include.get_str() + "\n"; + success = false; } } - return true; + return success; } -unsigned int ArgsManager::FlagsOfKnownArg(const std::string& key) const +Optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) const { - assert(key[0] == '-'); - - size_t option_index = key.find('.'); - if (option_index == std::string::npos) { - option_index = 1; - } else { - ++option_index; - } - if (key.substr(option_index, 2) == "no") { - option_index += 2; - } - - const std::string base_arg_name = '-' + key.substr(option_index); - LOCK(cs_args); for (const auto& arg_map : m_available_args) { - const auto search = arg_map.second.find(base_arg_name); + const auto search = arg_map.second.find(name); if (search != arg_map.second.end()) { return search->second.m_flags; } } - return ArgsManager::NONE; + return nullopt; } std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { - std::vector<std::string> result = {}; - if (IsArgNegated(strArg)) return result; // special case - - LOCK(cs_args); - - ArgsManagerHelper::AddArgs(result, m_override_args, strArg); - if (!m_network.empty()) { - ArgsManagerHelper::AddArgs(result, m_config_args, ArgsManagerHelper::NetworkArg(*this, strArg)); - } - - if (ArgsManagerHelper::UseDefaultSection(*this, strArg)) { - ArgsManagerHelper::AddArgs(result, m_config_args, strArg); + std::vector<std::string> result; + for (const util::SettingsValue& value : GetSettingsList(strArg)) { + result.push_back(value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str()); } - return result; } bool ArgsManager::IsArgSet(const std::string& strArg) const { - if (IsArgNegated(strArg)) return true; // special case - return ArgsManagerHelper::GetArg(*this, strArg).first; + return !GetSetting(strArg).isNull(); } bool ArgsManager::IsArgNegated(const std::string& strArg) const { - LOCK(cs_args); - - const auto& ov = m_override_args.find(strArg); - if (ov != m_override_args.end()) return ov->second.empty(); - - if (!m_network.empty()) { - const auto& cfs = m_config_args.find(ArgsManagerHelper::NetworkArg(*this, strArg)); - if (cfs != m_config_args.end()) return cfs->second.empty(); - } - - const auto& cf = m_config_args.find(strArg); - if (cf != m_config_args.end()) return cf->second.empty(); - - return false; + return GetSetting(strArg).isFalse(); } std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) const { - if (IsArgNegated(strArg)) return "0"; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return found_res.second; - return strDefault; + const util::SettingsValue value = GetSetting(strArg); + return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str(); } int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) const { - if (IsArgNegated(strArg)) return 0; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return atoi64(found_res.second); - return nDefault; + const util::SettingsValue value = GetSetting(strArg); + return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : atoi64(value.get_str()); } bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const { - if (IsArgNegated(strArg)) return false; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return InterpretBool(found_res.second); - return fDefault; + const util::SettingsValue value = GetSetting(strArg); + return value.isNull() ? fDefault : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); } bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) @@ -544,7 +408,7 @@ bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); - m_override_args[strArg] = {strValue}; + m_settings.forced_settings[SettingName(strArg)] = strValue; } void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) @@ -688,7 +552,7 @@ void PrintExceptionContinue(const std::exception* pex, const char* pszThread) { std::string message = FormatException(pex, pszThread); LogPrintf("\n\n************************\n%s\n", message); - tfm::format(std::cerr, "\n\n************************\n%s\n", message.c_str()); + tfm::format(std::cerr, "\n\n************************\n%s\n", message); } fs::path GetDefaultDataDir() @@ -797,16 +661,6 @@ fs::path GetConfigFile(const std::string& confPath) return AbsPathForConfigVal(fs::path(confPath), false); } -static std::string TrimString(const std::string& str, const std::string& pattern) -{ - std::string::size_type front = str.find_first_not_of(pattern); - if (front == std::string::npos) { - return std::string(); - } - std::string::size_type end = str.find_last_not_of(pattern); - return str.substr(front, end - front + 1); -} - static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; @@ -860,17 +714,20 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file return false; } for (const std::pair<std::string, std::string>& option : options) { - const std::string strKey = std::string("-") + option.first; - const unsigned int flags = FlagsOfKnownArg(strKey); + std::string section; + std::string key = option.first; + util::SettingsValue value = InterpretOption(section, key, option.second); + Optional<unsigned int> flags = GetArgFlags('-' + key); if (flags) { - if (!InterpretOption(strKey, option.second, flags, m_config_args, error)) { + if (!CheckValid(key, value, *flags, error)) { return false; } + m_settings.ro_config[section][key].push_back(value); } else { if (ignore_invalid_keys) { LogPrintf("Ignoring unknown configuration value %s\n", option.first); } else { - error = strprintf("Invalid configuration value %s", option.first.c_str()); + error = strprintf("Invalid configuration value %s", option.first); return false; } } @@ -882,7 +739,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) { { LOCK(cs_args); - m_config_args.clear(); + m_settings.ro_config.clear(); m_config_sections.clear(); } @@ -894,58 +751,64 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { return false; } - // if there is an -includeconf in the override args, but it is empty, that means the user - // passed '-noincludeconf' on the command line, in which case we should not include anything - bool emptyIncludeConf; + // `-includeconf` cannot be included in the command line arguments except + // as `-noincludeconf` (which indicates that no included conf file should be used). + bool use_conf_file{true}; { LOCK(cs_args); - emptyIncludeConf = m_override_args.count("-includeconf") == 0; + if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + // ParseParameters() fails if a non-negated -includeconf is passed on the command-line + assert(util::SettingsSpan(*includes).last_negated()); + use_conf_file = false; + } } - if (emptyIncludeConf) { + if (use_conf_file) { std::string chain_id = GetChainName(); - std::vector<std::string> includeconf(GetArgs("-includeconf")); - { - // We haven't set m_network yet (that happens in SelectParams()), so manually check - // for network.includeconf args. - std::vector<std::string> includeconf_net(GetArgs(std::string("-") + chain_id + ".includeconf")); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - } + std::vector<std::string> conf_file_names; - // Remove -includeconf from configuration, so we can warn about recursion - // later - { + auto add_includes = [&](const std::string& network, size_t skip = 0) { + size_t num_values = 0; LOCK(cs_args); - m_config_args.erase("-includeconf"); - m_config_args.erase(std::string("-") + chain_id + ".includeconf"); - } - - for (const std::string& to_include : includeconf) { - fsbridge::ifstream include_config(GetConfigFile(to_include)); - if (include_config.good()) { - if (!ReadConfigStream(include_config, to_include, error, ignore_invalid_keys)) { + if (auto* section = util::FindKey(m_settings.ro_config, network)) { + if (auto* values = util::FindKey(*section, "includeconf")) { + for (size_t i = std::max(skip, util::SettingsSpan(*values).negated()); i < values->size(); ++i) { + conf_file_names.push_back((*values)[i].get_str()); + } + num_values = values->size(); + } + } + return num_values; + }; + + // We haven't set m_network yet (that happens in SelectParams()), so manually check + // for network.includeconf args. + const size_t chain_includes = add_includes(chain_id); + const size_t default_includes = add_includes({}); + + for (const std::string& conf_file_name : conf_file_names) { + fsbridge::ifstream conf_file_stream(GetConfigFile(conf_file_name)); + if (conf_file_stream.good()) { + if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; } - LogPrintf("Included configuration file %s\n", to_include.c_str()); + LogPrintf("Included configuration file %s\n", conf_file_name); } else { - error = "Failed to include configuration file " + to_include; + error = "Failed to include configuration file " + conf_file_name; return false; } } // Warn about recursive -includeconf - includeconf = GetArgs("-includeconf"); - { - std::vector<std::string> includeconf_net(GetArgs(std::string("-") + chain_id + ".includeconf")); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - std::string chain_id_final = GetChainName(); - if (chain_id_final != chain_id) { - // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs - includeconf_net = GetArgs(std::string("-") + chain_id_final + ".includeconf"); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - } + conf_file_names.clear(); + add_includes(chain_id, /* skip= */ chain_includes); + add_includes({}, /* skip= */ default_includes); + std::string chain_id_final = GetChainName(); + if (chain_id_final != chain_id) { + // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs + add_includes(chain_id_final); } - for (const std::string& to_include : includeconf) { - tfm::format(std::cerr, "warning: -includeconf cannot be used from included files; ignoring -includeconf=%s\n", to_include.c_str()); + for (const std::string& conf_file_name : conf_file_names) { + tfm::format(std::cerr, "warning: -includeconf cannot be used from included files; ignoring -includeconf=%s\n", conf_file_name); } } } @@ -953,7 +816,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) // If datadir is changed in .conf file: ClearDatadirCache(); if (!CheckDataDirOption()) { - error = strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "").c_str()); + error = strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "")); return false; } return true; @@ -961,9 +824,16 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) std::string ArgsManager::GetChainName() const { - LOCK(cs_args); - const bool fRegTest = ArgsManagerHelper::GetNetBoolArg(*this, "-regtest"); - const bool fTestNet = ArgsManagerHelper::GetNetBoolArg(*this, "-testnet"); + auto get_net = [&](const std::string& arg) { + LOCK(cs_args); + util::SettingsValue value = util::GetSetting(m_settings, /* section= */ "", SettingName(arg), + /* ignore_default_section_config= */ false, + /* get_chain_name= */ true); + return value.isNull() ? false : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); + }; + + const bool fRegTest = get_net("-regtest"); + const bool fTestNet = get_net("-testnet"); const bool is_chain_arg_set = IsArgSet("-chain"); if ((int)is_chain_arg_set + (int)fRegTest + (int)fTestNet > 1) { @@ -976,6 +846,50 @@ std::string ArgsManager::GetChainName() const return GetArg("-chain", CBaseChainParams::MAIN); } +bool ArgsManager::UseDefaultSection(const std::string& arg) const +{ + return m_network == CBaseChainParams::MAIN || m_network_only_args.count(arg) == 0; +} + +util::SettingsValue ArgsManager::GetSetting(const std::string& arg) const +{ + LOCK(cs_args); + return util::GetSetting( + m_settings, m_network, SettingName(arg), !UseDefaultSection(arg), /* get_chain_name= */ false); +} + +std::vector<util::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const +{ + LOCK(cs_args); + return util::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); +} + +void ArgsManager::logArgsPrefix( + const std::string& prefix, + const std::string& section, + const std::map<std::string, std::vector<util::SettingsValue>>& args) const +{ + std::string section_str = section.empty() ? "" : "[" + section + "] "; + for (const auto& arg : args) { + for (const auto& value : arg.second) { + Optional<unsigned int> flags = GetArgFlags('-' + arg.first); + if (flags) { + std::string value_str = (*flags & SENSITIVE) ? "****" : value.write(); + LogPrintf("%s %s%s=%s\n", prefix, section_str, arg.first, value_str); + } + } + } +} + +void ArgsManager::LogArgs() const +{ + LOCK(cs_args); + for (const auto& section : m_settings.ro_config) { + logArgsPrefix("Config file arg:", section.first, section.second); + } + logArgsPrefix("Command-line arg:", "", m_settings.command_line_options); +} + bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 @@ -1086,17 +1000,19 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { SetEndOfFile(hFile); #elif defined(MAC_OSX) // OSX specific version + // NOTE: Contrary to other OS versions, the OSX version assumes that + // NOTE: offset is the size of the file. fstore_t fst; fst.fst_flags = F_ALLOCATECONTIG; fst.fst_posmode = F_PEOFPOSMODE; fst.fst_offset = 0; - fst.fst_length = (off_t)offset + length; + fst.fst_length = length; // mac os fst_length takes the # of free bytes to allocate, not desired file size fst.fst_bytesalloc = 0; if (fcntl(fileno(file), F_PREALLOCATE, &fst) == -1) { fst.fst_flags = F_ALLOCATEALL; fcntl(fileno(file), F_PREALLOCATE, &fst); } - ftruncate(fileno(file), fst.fst_length); + ftruncate(fileno(file), static_cast<off_t>(offset) + length); #else #if defined(__linux__) // Version using posix_fallocate @@ -1228,17 +1144,13 @@ fs::path AbsPathForConfigVal(const fs::path& path, bool net_specific) return fs::absolute(path, GetDataDir(net_specific)); } -int ScheduleBatchPriority() +void ScheduleBatchPriority() { #ifdef SCHED_BATCH const static sched_param param{}; - if (int ret = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m)) { + if (pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m) != 0) { LogPrintf("Failed to pthread_setschedparam: %s\n", strerror(errno)); - return ret; } - return 0; -#else - return 1; #endif } diff --git a/src/util/system.h b/src/util/system.h index 908a3c407d..bb69181de9 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,9 +19,11 @@ #include <compat/assumptions.h> #include <fs.h> #include <logging.h> +#include <optional.h> #include <sync.h> #include <tinyformat.h> #include <util/memory.h> +#include <util/settings.h> #include <util/threadnames.h> #include <util/time.h> @@ -131,7 +133,6 @@ class ArgsManager { public: enum Flags { - NONE = 0x00, // Boolean options can accept negation syntax -noOPTION or -noOPTION=1 ALLOW_BOOL = 0x01, ALLOW_INT = 0x02, @@ -144,11 +145,11 @@ public: * between mainnet and regtest/testnet won't cause problems due to these * parameters by accident. */ NETWORK_ONLY = 0x200, + // This argument's value is sensitive (such as a password). + SENSITIVE = 0x400, }; protected: - friend class ArgsManagerHelper; - struct Arg { std::string m_help_param; @@ -156,9 +157,8 @@ protected: unsigned int m_flags; }; - mutable CCriticalSection cs_args; - std::map<std::string, std::vector<std::string>> m_override_args GUARDED_BY(cs_args); - std::map<std::string, std::vector<std::string>> m_config_args GUARDED_BY(cs_args); + mutable RecursiveMutex cs_args; + util::Settings m_settings GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); @@ -166,6 +166,27 @@ protected: NODISCARD bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); + /** + * Returns true if settings values from the default section should be used, + * depending on the current network and whether the setting is + * network-specific. + */ + bool UseDefaultSection(const std::string& arg) const EXCLUSIVE_LOCKS_REQUIRED(cs_args); + + /** + * Get setting value. + * + * Result will be null if setting was unset, true if "-setting" argument was passed + * false if "-nosetting" argument was passed, and a string if a "-setting=value" + * argument was passed. + */ + util::SettingsValue GetSetting(const std::string& arg) const; + + /** + * Get list of setting values. + */ + std::vector<util::SettingsValue> GetSettingsList(const std::string& arg) const; + public: ArgsManager(); @@ -265,7 +286,7 @@ public: void ForceSetArg(const std::string& strArg, const std::string& strValue); /** - * Looks for -regtest, -testnet and returns the appropriate BIP70 chain name. + * Returns the appropriate chain name from the program arguments. * @return CBaseChainParams::MAIN by default; raises runtime error if an invalid combination is given. */ std::string GetChainName() const; @@ -296,9 +317,22 @@ public: /** * Return Flags for known arg. - * Return ArgsManager::NONE for unknown arg. + * Return nullopt for unknown arg. + */ + Optional<unsigned int> GetArgFlags(const std::string& name) const; + + /** + * Log the config file options and the command line arguments, + * useful for troubleshooting. */ - unsigned int FlagsOfKnownArg(const std::string& key) const; + void LogArgs() const; + +private: + // Helper function for LogArgs(). + void logArgsPrefix( + const std::string& prefix, + const std::string& section, + const std::map<std::string, std::vector<util::SettingsValue>>& args) const; }; extern ArgsManager gArgs; @@ -367,10 +401,8 @@ std::string CopyrightHolders(const std::string& strPrefix); * On platforms that support it, tell the kernel the calling thread is * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. * - * @return The return value of sched_setschedule(), or 1 on systems without - * sched_setschedule(). */ -int ScheduleBatchPriority(); +void ScheduleBatchPriority(); namespace util { diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index 20df403a66..764fffabd7 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/threadnames.h b/src/util/threadnames.h index 69a1b55bfe..64b2689cf1 100644 --- a/src/util/threadnames.h +++ b/src/util/threadnames.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/time.cpp b/src/util/time.cpp index 2b202ae95f..2afff2626b 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -111,3 +111,17 @@ std::string FormatISO8601Date(int64_t nTime) { #endif return strprintf("%04i-%02i-%02i", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday); } + +int64_t ParseISO8601DateTime(const std::string& str) +{ + static const boost::posix_time::ptime epoch = boost::posix_time::from_time_t(0); + static const std::locale loc(std::locale::classic(), + new boost::posix_time::time_input_facet("%Y-%m-%dT%H:%M:%SZ")); + std::istringstream iss(str); + iss.imbue(loc); + boost::posix_time::ptime ptime(boost::date_time::not_a_date_time); + iss >> ptime; + if (ptime.is_not_a_date_time() || epoch > ptime) + return 0; + return (ptime - epoch).total_seconds(); +}
\ No newline at end of file diff --git a/src/util/time.h b/src/util/time.h index c0470a2136..af4390aa1c 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -48,5 +48,6 @@ T GetTime(); */ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); +int64_t ParseISO8601DateTime(const std::string& str); #endif // BITCOIN_UTIL_TIME_H diff --git a/src/util/translation.h b/src/util/translation.h index 0e6eb5a094..fc45da440a 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -6,7 +6,7 @@ #define BITCOIN_UTIL_TRANSLATION_H #include <tinyformat.h> - +#include <functional> /** * Bilingual messages: diff --git a/src/util/url.cpp b/src/util/url.cpp index 49eacbf2d0..e42d93bce8 100644 --- a/src/util/url.cpp +++ b/src/util/url.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/url.h b/src/util/url.h index 3d7315a338..e9ea2ab765 100644 --- a/src/util/url.h +++ b/src/util/url.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/validation.cpp b/src/util/validation.cpp index 9a0d889447..89bc6665a4 100644 --- a/src/util/validation.cpp +++ b/src/util/validation.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,12 +8,18 @@ #include <consensus/validation.h> #include <tinyformat.h> -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state) +std::string FormatStateMessage(const ValidationState &state) { - return strprintf("%s%s", - state.GetRejectReason(), - state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage()); + if (state.IsValid()) { + return "Valid"; + } + + const std::string debug_message = state.GetDebugMessage(); + if (!debug_message.empty()) { + return strprintf("%s, %s", state.GetRejectReason(), debug_message); + } + + return state.GetRejectReason(); } const std::string strMessageMagic = "Bitcoin Signed Message:\n"; diff --git a/src/util/validation.h b/src/util/validation.h index 32559853ee..da2cf9f102 100644 --- a/src/util/validation.h +++ b/src/util/validation.h @@ -8,10 +8,10 @@ #include <string> -class CValidationState; +class ValidationState; -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state); +/** Convert ValidationState to a human-readable message for logging */ +std::string FormatStateMessage(const ValidationState &state); extern const std::string strMessageMagic; diff --git a/src/validation.cpp b/src/validation.cpp index 9301066c6a..bab04b8e34 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,6 +18,8 @@ #include <flatfile.h> #include <hash.h> #include <index/txindex.h> +#include <logging.h> +#include <logging/timer.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -107,7 +109,7 @@ CBlockIndex *pindexBestHeader = nullptr; Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; -int nScriptCheckThreads = 0; +bool g_parallel_script_checks{false}; std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); bool fHavePruned = false; @@ -127,14 +129,11 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CBlockPolicyEstimator feeEstimator; CTxMemPool mempool(&feeEstimator); -/** Constant stuff for coinbase transactions we create: */ -CScript COINBASE_FLAGS; - // Internal stuff namespace { CBlockIndex* pindexBestInvalid = nullptr; - CCriticalSection cs_LastBlockFile; + RecursiveMutex cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; int nLastBlockFile = 0; /** Global flag to indicate we should check to see if there are @@ -181,7 +180,7 @@ std::unique_ptr<CBlockTreeDB> pblocktree; // See definition for documentation static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); @@ -363,9 +362,9 @@ static void UpdateMempoolForReorg(DisconnectedBlockTransactions& disconnectpool, auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions - CValidationState stateDummy; + TxValidationState stateDummy; if (!fAddToMempool || (*it)->IsCoinBase() || - !AcceptToMemoryPool(mempool, stateDummy, *it, nullptr /* pfMissingInputs */, + !AcceptToMemoryPool(mempool, stateDummy, *it, nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */)) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). @@ -391,25 +390,25 @@ static void UpdateMempoolForReorg(DisconnectedBlockTransactions& disconnectpool, // Used to avoid mempool polluting consensus critical paths if CCoinsViewMempool // were somehow broken and returning the wrong scriptPubKeys -static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, +static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, unsigned int flags, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); // pool.cs should be locked already, but go ahead and re-take the lock here // to enforce that mempool doesn't change between when we check the view - // and when we actually call through to CheckInputs + // and when we actually call through to CheckInputScripts LOCK(pool.cs); assert(!tx.IsCoinBase()); for (const CTxIn& txin : tx.vin) { const Coin& coin = view.AccessCoin(txin.prevout); - // At this point we haven't actually checked if the coins are all - // available (or shouldn't assume we have, since CheckInputs does). - // So we just return failure if the inputs are not available here, - // and then only have to check equivalence for available inputs. + // AcceptToMemoryPoolWorker has already checked that the coins are + // available, so this shouldn't fail. If the inputs are not available + // here then return false. if (coin.IsSpent()) return false; + // Check equivalence for available inputs. const CTransactionRef& txFrom = pool.get(txin.prevout.hash); if (txFrom) { assert(txFrom->GetHash() == txin.prevout.hash); @@ -422,8 +421,8 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, CValidationSt } } - // Call CheckInputs() to cache signature and script validity against current tip consensus rules. - return CheckInputs(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); + // Call CheckInputScripts() to cache signature and script validity against current tip consensus rules. + return CheckInputScripts(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); } namespace { @@ -441,8 +440,7 @@ public: // around easier. struct ATMPArgs { const CChainParams& m_chainparams; - CValidationState &m_state; - bool* m_missing_inputs; + TxValidationState &m_state; const int64_t m_accept_time; std::list<CTransactionRef>* m_replaced_transactions; const bool m_bypass_limits; @@ -502,15 +500,15 @@ private: bool Finalize(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Compare a package's feerate against minimum allowed. - bool CheckFeeRate(size_t package_size, CAmount package_fee, CValidationState& state) + bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) { CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } if (package_fee < ::minRelayTxFee.GetFee(package_size)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "min relay fee not met", strprintf("%d < %d", package_fee, ::minRelayTxFee.GetFee(package_size))); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", package_fee, ::minRelayTxFee.GetFee(package_size))); } return true; } @@ -537,8 +535,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const uint256& hash = ws.m_hash; // Copy/alias what we need out of args - CValidationState &state = args.m_state; - bool* pfMissingInputs = args.m_missing_inputs; + TxValidationState &state = args.m_state; const int64_t nAcceptTime = args.m_accept_time; const bool bypass_limits = args.m_bypass_limits; const CAmount& nAbsurdFee = args.m_absurd_fee; @@ -554,38 +551,34 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount& nConflictingFees = ws.m_conflicting_fees; size_t& nConflictingSize = ws.m_conflicting_size; - if (pfMissingInputs) { - *pfMissingInputs = false; - } - if (!CheckTransaction(tx, state)) return false; // state filled in by CheckTransaction // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "coinbase"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "coinbase"); // Rather not work on nonstandard transactions (unless -testnet/-regtest) std::string reason; if (fRequireStandard && !IsStandardTx(tx, reason)) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, reason); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, reason); // Do not work on transactions that are too small. // A transaction with 1 segwit input and 1 P2WPHK output has non-witness size of 82 bytes. // Transactions smaller than this are not relayed to mitigate CVE-2017-12842 by not relaying // 64-byte transactions. if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, "tx-size-small"); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) - return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, "non-final"); + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); // is it already in the memory pool? if (m_pool.exists(hash)) { - return state.Invalid(ValidationInvalidReason::TX_CONFLICT, false, "txn-already-in-mempool"); + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); } // Check for conflicts with in-memory transactions @@ -617,7 +610,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } if (fReplacementOptOut) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "txn-mempool-conflict"); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } setConflicts.insert(ptxConflicting->GetHash()); @@ -643,14 +636,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) for (size_t out = 0; out < tx.vout.size(); out++) { // Optimistically just do efficient check of cache for outputs if (coins_cache.HaveCoinInCache(COutPoint(hash, out))) { - return state.Invalid(ValidationInvalidReason::TX_CONFLICT, false, "txn-already-known"); + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-known"); } } // Otherwise assume this might be an orphan tx for which we just haven't seen parents yet - if (pfMissingInputs) { - *pfMissingInputs = true; - } - return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() + return state.Invalid(TxValidationResult::TX_MISSING_INPUTS, "bad-txns-inputs-missingorspent"); } } @@ -668,7 +658,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Must keep pool.cs for this unless we change CheckSequenceLocks to take a // CoinsViewCache instead of create its own if (!CheckSequenceLocks(m_pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) - return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, "non-BIP68-final"); + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); CAmount nFees = 0; if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { @@ -677,11 +667,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Check for non-standard pay-to-script-hash in inputs if (fRequireStandard && !AreInputsStandard(tx, m_view)) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, "bad-txns-nonstandard-inputs"); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-nonstandard-inputs"); // Check for non-standard witness in P2WSH if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, m_view)) - return state.Invalid(ValidationInvalidReason::TX_WITNESS_MUTATED, false, "bad-witness-nonstandard"); + return state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, "bad-witness-nonstandard"); int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); @@ -705,7 +695,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) unsigned int nSize = entry->GetTxSize(); if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, "bad-txns-too-many-sigops", + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", strprintf("%d", nSigOpsCost)); // No transactions are allowed below minRelayTxFee except from disconnected @@ -713,7 +703,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!bypass_limits && !CheckFeeRate(nSize, nModifiedFees, state)) return false; if (nAbsurdFee && nFees > nAbsurdFee) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "absurdly-high-fee", strprintf("%d > %d", nFees, nAbsurdFee)); const CTxMemPool::setEntries setIterConflicting = m_pool.GetIterSet(setConflicts); @@ -742,7 +732,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // To check these we first check if we meet the RBF criteria, above, and increment the descendant // limits by the direct conflict and its descendants (as these are recalculated in // CalculateMempoolAncestors by assuming the new transaction being added is a new descendant, with no - // removals, of each parent's existing dependant set). The ancestor count limits are unmodified (as + // removals, of each parent's existing dependent set). The ancestor count limits are unmodified (as // the ancestor limits should be the same for both our new transaction and any conflicts). // We don't bother incrementing m_limit_descendants by the full removal count as that limit never comes // into force here (as we're only adding a single transaction). @@ -771,7 +761,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // this, see https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html if (nSize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || !m_pool.CalculateMemPoolAncestors(*entry, setAncestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "too-long-mempool-chain", errString); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", errString); } } @@ -784,7 +774,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); if (setConflicts.count(hashAncestor)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-spends-conflicting-tx", + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", strprintf("%s spends conflicting transaction %s", hash.ToString(), hashAncestor.ToString())); @@ -824,7 +814,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); if (newFeeRate <= oldFeeRate) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "insufficient fee", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", hash.ToString(), newFeeRate.ToString(), @@ -852,7 +842,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) nConflictingSize += it->GetTxSize(); } } else { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "too many potential replacements", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too many potential replacements", strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", hash.ToString(), nConflictingCount, @@ -876,7 +866,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // it's cheaper to just check if the new input refers to a // tx that's in the mempool. if (m_pool.exists(tx.vin[j].prevout.hash)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "replacement-adds-unconfirmed", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "replacement-adds-unconfirmed", strprintf("replacement %s adds unconfirmed input, idx %d", hash.ToString(), j)); } @@ -888,7 +878,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // transactions would not be paid for. if (nModifiedFees < nConflictingFees) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "insufficient fee", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); } @@ -898,7 +888,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount nDeltaFees = nModifiedFees - nConflictingFees; if (nDeltaFees < ::incrementalRelayFee.GetFee(nSize)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "insufficient fee", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", hash.ToString(), FormatMoney(nDeltaFees), @@ -912,25 +902,24 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, Precompute { const CTransaction& tx = *ws.m_ptx; - CValidationState &state = args.m_state; + TxValidationState &state = args.m_state; constexpr unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; - // Check against previous transactions + // Check input scripts and signatures. // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputs(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { + if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we // need to turn both off, and compare against just turning off CLEANSTACK // to see if the failure is specifically due to witness validation. - CValidationState stateDummy; // Want reported failures to be from first CheckInputs - if (!tx.HasWitness() && CheckInputs(tx, stateDummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && - !CheckInputs(tx, stateDummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { + TxValidationState state_dummy; // Want reported failures to be from first CheckInputScripts + if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && + !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { // Only the witness is missing, so the transaction itself may be fine. - state.Invalid(ValidationInvalidReason::TX_WITNESS_MUTATED, false, + state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, state.GetRejectReason(), state.GetDebugMessage()); } - assert(IsTransactionReason(state.GetReason())); - return false; // state filled in by CheckInputs + return false; // state filled in by CheckInputScripts } return true; @@ -941,7 +930,7 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, Precomp const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - CValidationState &state = args.m_state; + TxValidationState &state = args.m_state; const CChainParams& chainparams = args.m_chainparams; // Check again against the current block tip's script verification @@ -961,7 +950,7 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, Precomp // transactions into the mempool can be exploited as a DoS attack. unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(::ChainActive().Tip(), chainparams.GetConsensus()); if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata)) { - return error("%s: BUG! PLEASE REPORT THIS! CheckInputs failed against latest-block but not STANDARD flags %s, %s", + return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } @@ -972,7 +961,7 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - CValidationState &state = args.m_state; + TxValidationState &state = args.m_state; const bool bypass_limits = args.m_bypass_limits; CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; @@ -1010,7 +999,7 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) if (!bypass_limits) { LimitMempoolSize(m_pool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); if (!m_pool.exists(hash)) - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, "mempool full"); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } return true; } @@ -1047,12 +1036,12 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs } // anon namespace /** (try to) add transaction to memory pool with a specified acceptance time **/ -static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, - bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, +static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, state, pfMissingInputs, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept }; + MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept }; bool res = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); if (!res) { // Remove coins that were not present in the coins cache before calling ATMPW; @@ -1064,17 +1053,17 @@ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPo ::ChainstateActive().CoinsTip().Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits - CValidationState stateDummy; - ::ChainstateActive().FlushStateToDisk(chainparams, stateDummy, FlushStateMode::PERIODIC); + BlockValidationState state_dummy; + ::ChainstateActive().FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); return res; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, - bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced, +bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) { const CChainParams& chainparams = Params(); - return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, pfMissingInputs, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); + return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); } /** @@ -1403,10 +1392,14 @@ static void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) E CheckForkWarningConditions(); } +// Called both upon regular invalid block discovery *and* InvalidateBlock void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) pindexBestInvalid = pindexNew; + if (pindexBestHeader != nullptr && pindexBestHeader->GetAncestor(pindexNew->nHeight) == pindexNew) { + pindexBestHeader = ::ChainActive().Tip(); + } LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, @@ -1419,8 +1412,10 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c CheckForkWarningConditions(); } -void CChainState::InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { - if (state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { +// Same as InvalidChainFound, above, except not called directly from InvalidateBlock, +// which does its own setBlockIndexCandidates manageent. +void CChainState::InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) { + if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; m_blockman.m_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); @@ -1477,8 +1472,10 @@ void InitScriptExecutionCache() { } /** - * Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts) - * This does not modify the UTXO set. + * Check whether all of this transaction's input scripts succeed. + * + * This involves ECDSA signature checks so can be computationally intensive. This function should + * only be called after the cheap sanity checks in CheckTxInputs passed. * * If pvChecks is not nullptr, script checks are pushed onto it instead of being performed inline. Any * script checks which are not necessary (eg due to script execution cache hits) are, obviously, @@ -1493,7 +1490,7 @@ void InitScriptExecutionCache() { * * Non-static (and re-declared) in src/test/txvalidationcache_tests.cpp */ -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (tx.IsCoinBase()) return true; @@ -1545,10 +1542,10 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi CScriptCheck check2(coin.out, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheSigStore, &txdata); if (check2()) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } // MANDATORY flag failures correspond to - // ValidationInvalidReason::CONSENSUS. Because CONSENSUS + // TxValidationResult::TX_CONSENSUS. Because CONSENSUS // failures are the most serious case of validation // failures, we may need to consider using // RECENT_CONSENSUS_CHANGE for any script failure that @@ -1556,7 +1553,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // support, to avoid splitting the network (but this // depends on the details of how net_processing handles // such errors). - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); + return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } @@ -1641,7 +1638,7 @@ static bool AbortNode(const std::string& strMessage, const std::string& userMess return false; } -static bool AbortNode(CValidationState& state, const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) +static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) { AbortNode(strMessage, userMessage, prefix); return state.Error(strMessage); @@ -1755,9 +1752,9 @@ void static FlushBlockFile(bool fFinalize = false) } } -static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); +static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); -static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { @@ -1896,7 +1893,7 @@ static int64_t nBlocksTotal = 0; /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ -bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, +bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { AssertLockHeld(cs_main); @@ -1918,7 +1915,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // re-enforce that rule here (at least until we make it impossible for // GetAdjustedTime() to go backward). if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) { - if (state.GetReason() == ValidationInvalidReason::BLOCK_MUTATED) { + if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) { // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware // problems. @@ -2058,8 +2055,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl for (const auto& tx : block.vtx) { for (size_t o = 0; o < tx->vout.size(); o++) { if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): tried to overwrite transaction"), - "bad-txns-BIP30"); + LogPrintf("ERROR: ConnectBlock(): tried to overwrite transaction\n"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-BIP30"); } } } @@ -2079,7 +2076,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl CBlockUndo blockundo; - CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : nullptr); + CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); std::vector<int> prevheights; CAmount nFees = 0; @@ -2097,21 +2094,17 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl if (!tx.IsCoinBase()) { CAmount txfee = 0; - if (!Consensus::CheckTxInputs(tx, state, view, pindex->nHeight, txfee)) { - if (!IsBlockReason(state.GetReason())) { - // CheckTxInputs may return MISSING_INPUTS or - // PREMATURE_SPEND but we can't return that, as it's not - // defined for a block, so we reset the reason flag to - // CONSENSUS here. - state.Invalid(ValidationInvalidReason::CONSENSUS, false, - state.GetRejectReason(), state.GetDebugMessage()); - } + TxValidationState tx_state; + if (!Consensus::CheckTxInputs(tx, tx_state, view, pindex->nHeight, txfee)) { + // Any transaction validation failure in ConnectBlock is a block consensus failure + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, + tx_state.GetRejectReason(), tx_state.GetDebugMessage()); return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } nFees += txfee; if (!MoneyRange(nFees)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: accumulated fee in the block out of range.", __func__), - "bad-txns-accumulated-fee-outofrange"); + LogPrintf("ERROR: %s: accumulated fee in the block out of range.\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-accumulated-fee-outofrange"); } // Check that transaction is BIP68 final @@ -2123,8 +2116,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: contains a non-BIP68-final transaction", __func__), - "bad-txns-nonfinal"); + LogPrintf("ERROR: %s: contains a non-BIP68-final transaction\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal"); } } @@ -2133,27 +2126,22 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // * p2sh (when P2SH enabled in flags and excludes coinbase) // * witness (when witness enabled in flags and excludes coinbase) nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); - if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): too many sigops"), - "bad-blk-sigops"); + if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) { + LogPrintf("ERROR: ConnectBlock(): too many sigops\n"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); + } txdata.emplace_back(tx); if (!tx.IsCoinBase()) { std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ - if (fScriptChecks && !CheckInputs(tx, state, view, flags, fCacheResults, fCacheResults, txdata[i], nScriptCheckThreads ? &vChecks : nullptr)) { - if (state.GetReason() == ValidationInvalidReason::TX_NOT_STANDARD) { - // CheckInputs may return NOT_STANDARD for extra flags we passed, - // but we can't return that, as it's not defined for a block, so - // we reset the reason flag to CONSENSUS here. - // In the event of a future soft-fork, we may need to - // consider whether rewriting to CONSENSUS or - // RECENT_CONSENSUS_CHANGE would be more appropriate. - state.Invalid(ValidationInvalidReason::CONSENSUS, false, - state.GetRejectReason(), state.GetDebugMessage()); - } - return error("ConnectBlock(): CheckInputs on %s failed with %s", + TxValidationState tx_state; + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + // Any transaction validation failure in ConnectBlock is a block consensus failure + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, + tx_state.GetRejectReason(), tx_state.GetDebugMessage()); + return error("ConnectBlock(): CheckInputScripts on %s failed with %s", tx.GetHash().ToString(), FormatStateMessage(state)); } control.Add(vChecks); @@ -2169,14 +2157,15 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); - if (block.vtx[0]->GetValueOut() > blockReward) - return state.Invalid(ValidationInvalidReason::CONSENSUS, - error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", - block.vtx[0]->GetValueOut(), blockReward), - "bad-cb-amount"); - - if (!control.Wait()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: CheckQueue failed", __func__), "block-validation-failed"); + if (block.vtx[0]->GetValueOut() > blockReward) { + LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); + } + + if (!control.Wait()) { + LogPrintf("ERROR: %s: CheckQueue failed\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "block-validation-failed"); + } int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); @@ -2204,28 +2193,68 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl return true; } +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_pool) +{ + return this->GetCoinsCacheSizeState( + tx_pool, + nCoinCacheUsage, + gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); +} + +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( + const CTxMemPool& tx_pool, + size_t max_coins_cache_size_bytes, + size_t max_mempool_size_bytes) +{ + int64_t nMempoolUsage = tx_pool.DynamicMemoryUsage(); + int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); + int64_t nTotalSpace = + max_coins_cache_size_bytes + std::max<int64_t>(max_mempool_size_bytes - nMempoolUsage, 0); + + //! No need to periodic flush if at least this much space still available. + static constexpr int64_t MAX_BLOCK_COINSDB_USAGE_BYTES = 10 * 1024 * 1024; // 10MB + int64_t large_threshold = + std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE_BYTES); + + if (cacheSize > nTotalSpace) { + LogPrintf("Cache size (%s) exceeds total space (%s)\n", cacheSize, nTotalSpace); + return CoinsCacheSizeState::CRITICAL; + } else if (cacheSize > large_threshold) { + return CoinsCacheSizeState::LARGE; + } + return CoinsCacheSizeState::OK; +} + bool CChainState::FlushStateToDisk( const CChainParams& chainparams, - CValidationState &state, + BlockValidationState &state, FlushStateMode mode, int nManualPruneHeight) { - int64_t nMempoolUsage = mempool.DynamicMemoryUsage(); LOCK(cs_main); assert(this->CanFlushToDisk()); static int64_t nLastWrite = 0; static int64_t nLastFlush = 0; std::set<int> setFilesToPrune; bool full_flush_completed = false; + + const size_t coins_count = CoinsTip().GetCacheSize(); + const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage(); + try { { bool fFlushForPrune = false; bool fDoFullFlush = false; + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(::mempool); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { + LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH); + FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); } else { + LOG_TIME_MILLIS("find files to prune", BCLog::BENCH); + FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } @@ -2245,13 +2274,10 @@ bool CChainState::FlushStateToDisk( if (nLastFlush == 0) { nLastFlush = nNow; } - int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); - int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FlushStateMode::PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); + bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE; // The cache is over the limit, we have to write now. - bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cacheSize > nTotalSpace; + bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. @@ -2264,10 +2290,17 @@ bool CChainState::FlushStateToDisk( if (!CheckDiskSpace(GetBlocksDir())) { return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } - // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); + { + LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH); + + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + } + // Then update all block file information (which may refer to block and undo files). { + LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH); + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; vFiles.reserve(setDirtyFileInfo.size()); for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { @@ -2285,12 +2318,18 @@ bool CChainState::FlushStateToDisk( } } // Finally remove any pruned files - if (fFlushForPrune) + if (fFlushForPrune) { + LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH); + UnlinkPrunedFiles(setFilesToPrune); + } nLastWrite = nNow; } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { + LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000)); + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already @@ -2317,7 +2356,7 @@ bool CChainState::FlushStateToDisk( } void CChainState::ForceFlushStateToDisk() { - CValidationState state; + BlockValidationState state; const CChainParams& chainparams = Params(); if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, FormatStateMessage(state)); @@ -2325,7 +2364,7 @@ void CChainState::ForceFlushStateToDisk() { } void CChainState::PruneAndFlush() { - CValidationState state; + BlockValidationState state; fCheckForPruning = true; const CChainParams& chainparams = Params(); @@ -2411,7 +2450,7 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ -bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) +bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); @@ -2453,7 +2492,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - GetMainSignals().BlockDisconnected(pblock); + GetMainSignals().BlockDisconnected(pblock, pindexDelete); return true; } @@ -2531,7 +2570,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) +bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. @@ -2663,7 +2702,7 @@ void CChainState::PruneBlockIndexCandidates() { * * @returns true unless a system error occurred */ -bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); @@ -2710,10 +2749,10 @@ bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainPar if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. - if (state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { + if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { InvalidChainFound(vpindexToConnect.front()); } - state = CValidationState(); + state = BlockValidationState(); fInvalidFound = true; fContinue = false; break; @@ -2781,7 +2820,7 @@ static void LimitValidationInterfaceQueue() LOCKS_EXCLUDED(cs_main) { } } -bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set @@ -2881,11 +2920,11 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& return true; } -bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { return ::ChainstateActive().ActivateBestChain(state, chainparams, std::move(pblock)); } -bool CChainState::PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) +bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { LOCK(cs_main); @@ -2913,11 +2952,11 @@ bool CChainState::PreciousBlock(CValidationState& state, const CChainParams& par return ActivateBestChain(state, params, std::shared_ptr<const CBlock>()); } -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) { +bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { return ::ChainstateActive().PreciousBlock(state, params, pindex); } -bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { CBlockIndex* to_mark_failed = pindex; bool pindex_was_in_chain = false; @@ -3053,7 +3092,7 @@ bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& c return true; } -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { +bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { return ::ChainstateActive().InvalidateBlock(state, chainparams, pindex); } @@ -3227,7 +3266,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n return true; } -static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) +static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; @@ -3249,16 +3288,16 @@ static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, un return true; } -static bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) +static bool CheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, "high-hash", "proof of work failed"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "high-hash", "proof of work failed"); return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. @@ -3275,13 +3314,13 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, "bad-txnmrklroot", "hashMerkleRoot mismatch"); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-txnmrklroot", "hashMerkleRoot mismatch"); // Check for merkle tree malleability (CVE-2012-2459): repeating sequences // of transactions in a block without affecting the merkle root of a block, // while still invalidating it. if (mutated) - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, "bad-txns-duplicate", "duplicate transaction"); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-txns-duplicate", "duplicate transaction"); } // All potential-corruption validation must be done before we do any @@ -3292,28 +3331,34 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P // Size limits if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-blk-length", "size limits failed"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-cb-missing", "first tx is not coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-missing", "first tx is not coinbase"); for (unsigned int i = 1; i < block.vtx.size(); i++) if (block.vtx[i]->IsCoinBase()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-cb-multiple", "more than one coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-multiple", "more than one coinbase"); // Check transactions - for (const auto& tx : block.vtx) - if (!CheckTransaction(*tx, state)) - return state.Invalid(state.GetReason(), false, state.GetRejectReason(), - strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), state.GetDebugMessage())); - + // Must check for duplicate inputs (see CVE-2018-17144) + for (const auto& tx : block.vtx) { + TxValidationState tx_state; + if (!CheckTransaction(*tx, tx_state)) { + // CheckBlock() does context-free validation checks. The only + // possible failures are consensus failures. + assert(tx_state.GetResult() == TxValidationResult::TX_CONSENSUS); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), + strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), tx_state.GetDebugMessage())); + } + } unsigned int nSigOps = 0; for (const auto& tx : block.vtx) { nSigOps += GetLegacySigOpCount(*tx); } if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-blk-sigops", "out-of-bounds SigOpCount"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops", "out-of-bounds SigOpCount"); if (fCheckPOW && fCheckMerkleRoot) block.fChecked = true; @@ -3406,7 +3451,7 @@ static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOC * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { assert(pindexPrev != nullptr); const int nHeight = pindexPrev->nHeight + 1; @@ -3414,7 +3459,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta // Check proof of work const Consensus::Params& consensusParams = params.GetConsensus(); if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, "bad-diffbits", "incorrect proof of work"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "bad-diffbits", "incorrect proof of work"); // Check against checkpoints if (fCheckpointsEnabled) { @@ -3422,24 +3467,26 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // g_blockman.m_block_index. CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.Invalid(ValidationInvalidReason::BLOCK_CHECKPOINT, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), "bad-fork-prior-to-checkpoint"); + if (pcheckpoint && nHeight < pcheckpoint->nHeight) { + LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); + return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); + } } // Check timestamp against prev if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, "time-too-old", "block's timestamp is too early"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "time-too-old", "block's timestamp is too early"); // Check timestamp if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) - return state.Invalid(ValidationInvalidReason::BLOCK_TIME_FUTURE, false, "time-too-new", "block timestamp too far in the future"); + return state.Invalid(BlockValidationResult::BLOCK_TIME_FUTURE, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: // check for version 2, 3 and 4 upgrades if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, strprintf("bad-version(0x%08x)", block.nVersion), + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, strprintf("bad-version(0x%08x)", block.nVersion), strprintf("rejected nVersion=0x%08x block", block.nVersion)); return true; @@ -3451,7 +3498,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) +static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; @@ -3469,7 +3516,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // Check that all transactions are finalized for (const auto& tx : block.vtx) { if (!IsFinalTx(*tx, nHeight, nLockTimeCutoff)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-txns-nonfinal", "non-final transaction"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal", "non-final transaction"); } } @@ -3479,7 +3526,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c CScript expect = CScript() << nHeight; if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || !std::equal(expect.begin(), expect.end(), block.vtx[0]->vin[0].scriptSig.begin())) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-cb-height", "block height mismatch in coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-height", "block height mismatch in coinbase"); } } @@ -3501,11 +3548,11 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // already does not permit it, it is impossible to trigger in the // witness tree. if (block.vtx[0]->vin[0].scriptWitness.stack.size() != 1 || block.vtx[0]->vin[0].scriptWitness.stack[0].size() != 32) { - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, "bad-witness-nonce-size", strprintf("%s : invalid witness reserved value size", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-witness-nonce-size", strprintf("%s : invalid witness reserved value size", __func__)); } CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0]->vin[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); if (memcmp(hashWitness.begin(), &block.vtx[0]->vout[commitpos].scriptPubKey[6], 32)) { - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, "bad-witness-merkle-match", strprintf("%s : witness merkle commitment mismatch", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-witness-merkle-match", strprintf("%s : witness merkle commitment mismatch", __func__)); } fHaveWitness = true; } @@ -3515,7 +3562,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c if (!fHaveWitness) { for (const auto& tx : block.vtx) { if (tx->HasWitness()) { - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, "unexpected-witness", strprintf("%s : unexpected witness data found", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "unexpected-witness", strprintf("%s : unexpected witness data found", __func__)); } } } @@ -3527,13 +3574,13 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // the block hash, so we couldn't mark the block as permanently // failed). if (GetBlockWeight(block) > MAX_BLOCK_WEIGHT) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, "bad-blk-weight", strprintf("%s : weight limit failed", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-weight", strprintf("%s : weight limit failed", __func__)); } return true; } -bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate @@ -3546,8 +3593,10 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState pindex = miSelf->second; if (ppindex) *ppindex = pindex; - if (pindex->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(ValidationInvalidReason::CACHED_INVALID, error("%s: block %s is marked invalid", __func__, hash.ToString()), "duplicate"); + if (pindex->nStatus & BLOCK_FAILED_MASK) { + LogPrintf("ERROR: %s: block %s is marked invalid\n", __func__, hash.ToString()); + return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate"); + } return true; } @@ -3557,11 +3606,15 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState // Get prev block index CBlockIndex* pindexPrev = nullptr; BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); - if (mi == m_block_index.end()) - return state.Invalid(ValidationInvalidReason::BLOCK_MISSING_PREV, error("%s: prev block not found", __func__), "prev-blk-not-found"); + if (mi == m_block_index.end()) { + LogPrintf("ERROR: %s: prev block not found\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); + } pindexPrev = (*mi).second; - if (pindexPrev->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_PREV, error("%s: prev block invalid", __func__), "bad-prevblk"); + if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { + LogPrintf("ERROR: %s: prev block invalid\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); + } if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); @@ -3598,7 +3651,8 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState setDirtyBlockIndex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_PREV, error("%s: prev block invalid", __func__), "bad-prevblk"); + LogPrintf("ERROR: %s: prev block invalid\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } } } @@ -3613,9 +3667,8 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex, CBlockHeader *first_invalid) +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { - if (first_invalid != nullptr) first_invalid->SetNull(); { LOCK(cs_main); for (const CBlockHeader& header : headers) { @@ -3624,7 +3677,6 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { - if (first_invalid) *first_invalid = header; return false; } if (ppindex) { @@ -3660,7 +3712,7 @@ static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChai } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3710,8 +3762,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali if (!CheckBlock(block, state, chainparams.GetConsensus()) || !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { - assert(IsBlockReason(state.GetReason())); - if (state.IsInvalid() && state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { + if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); } @@ -3750,7 +3801,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons { CBlockIndex *pindex = nullptr; if (fNewBlock) *fNewBlock = false; - CValidationState state; + BlockValidationState state; // CheckBlock() does not support multi-threaded block validation because CBlock::fChecked can cause data race. // Therefore, the following critical section must include the CheckBlock() call as well. @@ -3771,14 +3822,14 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons NotifyHeaderTip(); - CValidationState state; // Only used to report errors, not invalidity - ignore it + BlockValidationState state; // Only used to report errors, not invalidity - ignore it if (!::ChainstateActive().ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed (%s)", __func__, FormatStateMessage(state)); return true; } -bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) { AssertLockHeld(cs_main); assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); @@ -3889,7 +3940,7 @@ static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPr /* This function is called from the RPC code for pruneblockchain */ void PruneBlockFilesManual(int nManualPruneHeight) { - CValidationState state; + BlockValidationState state; const CChainParams& chainparams = Params(); if (!::ChainstateActive().FlushStateToDisk( chainparams, state, FlushStateMode::NONE, nManualPruneHeight)) { @@ -4185,7 +4236,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; - CValidationState state; + BlockValidationState state; int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ for (pindex = ::ChainActive().Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { @@ -4429,7 +4480,7 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) } // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 - CValidationState state; + BlockValidationState state; // Loop until the tip is below nHeight, or we reach a pruned block. while (!ShutdownRequested()) { { @@ -4497,7 +4548,7 @@ bool RewindBlockIndex(const CChainParams& params) { // FlushStateToDisk can possibly read ::ChainActive(). Be conservative // and skip it here, we're about to -reindex-chainstate anyway, so // it'll get called a bunch real soon. - CValidationState state; + BlockValidationState state; if (!::ChainstateActive().FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", FormatStateMessage(state)); return false; @@ -4649,7 +4700,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // process in case the block isn't known yet CBlockIndex* pindex = LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { - CValidationState state; + BlockValidationState state; if (::ChainstateActive().AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { nLoaded++; } @@ -4663,7 +4714,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, chainparams)) { break; } @@ -4686,7 +4737,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); - CValidationState dummy; + BlockValidationState dummy; if (::ChainstateActive().AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; @@ -4963,10 +5014,10 @@ bool LoadMempool(CTxMemPool& pool) if (amountdelta) { pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } - CValidationState state; + TxValidationState state; if (nTime + nExpiryTimeout > nNow) { LOCK(cs_main); - AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nullptr /* pfMissingInputs */, nTime, + AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nTime, nullptr /* plTxnReplaced */, false /* bypass_limits */, 0 /* nAbsurdFee */, false /* test_accept */); if (state.IsValid()) { @@ -5072,7 +5123,7 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate; } - return pindex->nChainTx / fTxTotal; + return std::min<double>(pindex->nChainTx / fTxTotal, 1.0); } class CMainCleanup diff --git a/src/validation.h b/src/validation.h index d17a320a47..a5335edc43 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,8 +21,8 @@ #include <txmempool.h> // For CTxMemPool::cs #include <txdb.h> #include <versionbits.h> +#include <serialize.h> -#include <algorithm> #include <atomic> #include <map> #include <memory> @@ -32,6 +32,7 @@ #include <vector> class CChainState; +class BlockValidationState; class CBlockIndex; class CBlockTreeDB; class CBlockUndo; @@ -41,7 +42,7 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; -class CValidationState; +class TxValidationState; struct ChainTxData; struct DisconnectedBlockTransactions; @@ -75,8 +76,8 @@ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB -/** Maximum number of script-checking threads allowed */ -static const int MAX_SCRIPTCHECK_THREADS = 16; +/** Maximum number of dedicated script-checking threads allowed */ +static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; /** Number of blocks that can be requested at any given time from a single peer. */ @@ -135,8 +136,7 @@ struct BlockHasher size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } }; -extern CScript COINBASE_FLAGS; -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; @@ -145,7 +145,10 @@ extern std::condition_variable g_best_block_cv; extern uint256 g_best_block; extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; -extern int nScriptCheckThreads; +/** Whether there are dedicated script-checking threads running. + * False indicates all script checking is done on the main threadMessageHandler thread. + */ +extern bool g_parallel_script_checks; extern bool fRequireStandard; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; @@ -221,9 +224,8 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons * @param[out] state This may be set to an Error state if any error occurred processing them * @param[in] chainparams The params for the chain we want to connect to * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers - * @param[out] first_invalid First header that fails validation, if one exists */ -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); @@ -248,7 +250,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::P * May not be called with cs_main held. May not be called in a * validationinterface callback. */ -bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); +bool ActivateBestChain(BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ @@ -272,8 +274,8 @@ void PruneBlockFilesManual(int nManualPruneHeight); /** (try to) add transaction to memory pool * plTxnReplaced will be appended to with all transactions replaced from mempool **/ -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, - bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced, +bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get the BIP9 state for a given deployment at the current tip. */ @@ -368,10 +370,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); /** Functions for validating blocks and updating the block tree */ /** Context-independent validity checks */ -bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); +bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Check a block is completely valid from start to finish (only works on top of our current best block) */ -bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Check whether witness commitments are required for a block, and whether to enforce NULLDUMMY (BIP 147) rules. * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ @@ -488,7 +490,7 @@ public: */ bool AcceptBlockHeader( const CBlockHeader& block, - CValidationState& state, + BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; @@ -528,6 +530,15 @@ public: void InitCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; +enum class CoinsCacheSizeState +{ + //! The coins cache is in immediate need of a flush. + CRITICAL = 2, + //! The cache is at >= 90% capacity. + LARGE = 1, + OK = 0 +}; + /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -549,7 +560,7 @@ private: * Every received block is assigned a unique and increasing identifier, so we * know which one to give priority in case of a fork. */ - CCriticalSection cs_nBlockSequenceId; + RecursiveMutex cs_nBlockSequenceId; /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ int32_t nBlockSequenceId = 1; /** Decreasing counter (used by subsequent preciousblock calls). */ @@ -561,7 +572,7 @@ private: * the ChainState CriticalSection * A lock that must be held when modifying this ChainState - held in ActivateBestChain() */ - CCriticalSection m_cs_chainstate; + RecursiveMutex m_cs_chainstate; /** * Whether this chainstate is undergoing initial block download. @@ -652,7 +663,7 @@ public: */ bool FlushStateToDisk( const CChainParams& chainparams, - CValidationState &state, + BlockValidationState &state, FlushStateMode mode, int nManualPruneHeight = 0); @@ -678,23 +689,23 @@ public: * @returns true unless a system error occurred */ bool ActivateBestChain( - CValidationState& state, + BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) LOCKS_EXCLUDED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); - bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + bool ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. - bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); // Manual block validity manipulation: - bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); - bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Replay blocks that aren't fully applied to the database. */ @@ -719,11 +730,22 @@ public: /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Dictates whether we need to flush the cache to disk or not. + //! + //! @return the state of the size of the coins cache. + CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool& tx_pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + CoinsCacheSizeState GetCoinsCacheSizeState( + const CTxMemPool& tx_pool, + size_t max_coins_cache_size_bytes, + size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + private: - bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); - bool ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); - void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -738,10 +760,10 @@ private: * May not be called in a * validationinterface callback. */ -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); +bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); /** Mark a block as invalid. */ -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); +bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index cf4a529a6d..0f513c065f 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -1,15 +1,20 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <validationinterface.h> +#include <chain.h> +#include <consensus/validation.h> +#include <logging.h> #include <primitives/block.h> +#include <primitives/transaction.h> #include <scheduler.h> -#include <txmempool.h> +#include <util/validation.h> #include <future> +#include <unordered_map> #include <utility> #include <boost/signals2/signal.hpp> @@ -29,10 +34,10 @@ struct MainSignalsInstance { boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; - boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const std::shared_ptr<const CBlock>&, const CBlockIndex* pindex)> BlockDisconnected; boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> ChainStateFlushed; - boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; + boost::signals2::signal<void (const CBlock&, const BlockValidationState&)> BlockChecked; boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; // We are not allowed to assume the scheduler only runs in one thread, @@ -46,11 +51,6 @@ struct MainSignalsInstance { static CMainSignals g_signals; -// This map has to a separate global instead of a member of MainSignalsInstance, -// because RegisterWithMempoolSignals is currently called before RegisterBackgroundSignalScheduler, -// so MainSignalsInstance hasn't been created yet. -static std::unordered_map<CTxMemPool*, boost::signals2::scoped_connection> g_connNotifyEntryRemoved; - void CMainSignals::RegisterBackgroundSignalScheduler(CScheduler& scheduler) { assert(!m_internals); m_internals.reset(new MainSignalsInstance(&scheduler)); @@ -71,17 +71,6 @@ size_t CMainSignals::CallbacksPending() { return m_internals->m_schedulerClient.CallbacksPending(); } -void CMainSignals::RegisterWithMempoolSignals(CTxMemPool& pool) { - g_connNotifyEntryRemoved.emplace(std::piecewise_construct, - std::forward_as_tuple(&pool), - std::forward_as_tuple(pool.NotifyEntryRemoved.connect(std::bind(&CMainSignals::MempoolEntryRemoved, this, std::placeholders::_1, std::placeholders::_2))) - ); -} - -void CMainSignals::UnregisterWithMempoolSignals(CTxMemPool& pool) { - g_connNotifyEntryRemoved.erase(&pool); -} - CMainSignals& GetMainSignals() { return g_signals; @@ -92,7 +81,7 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { conns.UpdatedBlockTip = g_signals.m_internals->UpdatedBlockTip.connect(std::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); conns.TransactionAddedToMempool = g_signals.m_internals->TransactionAddedToMempool.connect(std::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, std::placeholders::_1)); conns.BlockConnected = g_signals.m_internals->BlockConnected.connect(std::bind(&CValidationInterface::BlockConnected, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); - conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1)); + conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.TransactionRemovedFromMempool = g_signals.m_internals->TransactionRemovedFromMempool.connect(std::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, std::placeholders::_1)); conns.ChainStateFlushed = g_signals.m_internals->ChainStateFlushed.connect(std::bind(&CValidationInterface::ChainStateFlushed, pwalletIn, std::placeholders::_1)); conns.BlockChecked = g_signals.m_internals->BlockChecked.connect(std::bind(&CValidationInterface::BlockChecked, pwalletIn, std::placeholders::_1, std::placeholders::_2)); @@ -126,52 +115,89 @@ void SyncWithValidationInterfaceQueue() { promise.get_future().wait(); } -void CMainSignals::MempoolEntryRemoved(CTransactionRef ptx, MemPoolRemovalReason reason) { - if (reason != MemPoolRemovalReason::BLOCK && reason != MemPoolRemovalReason::CONFLICT) { - m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { - m_internals->TransactionRemovedFromMempool(ptx); - }); - } -} +// Use a macro instead of a function for conditional logging to prevent +// evaluating arguments when logging is not enabled. +// +// NOTE: The lambda captures all local variables by value. +#define ENQUEUE_AND_LOG_EVENT(event, fmt, name, ...) \ + do { \ + auto local_name = (name); \ + LOG_EVENT("Enqueuing " fmt, local_name, __VA_ARGS__); \ + m_internals->m_schedulerClient.AddToProcessQueue([=] { \ + LOG_EVENT(fmt, local_name, __VA_ARGS__); \ + event(); \ + }); \ + } while (0) + +#define LOG_EVENT(fmt, ...) \ + LogPrint(BCLog::VALIDATION, fmt "\n", __VA_ARGS__) void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { // Dependencies exist that require UpdatedBlockTip events to be delivered in the order in which // the chain actually updates. One way to ensure this is for the caller to invoke this signal // in the same critical section where the chain is updated - m_internals->m_schedulerClient.AddToProcessQueue([pindexNew, pindexFork, fInitialDownload, this] { + auto event = [pindexNew, pindexFork, fInitialDownload, this] { m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: new block hash=%s fork block hash=%s (in IBD=%s)", __func__, + pindexNew->GetBlockHash().ToString(), + pindexFork ? pindexFork->GetBlockHash().ToString() : "null", + fInitialDownload); } void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + auto event = [ptx, this] { m_internals->TransactionAddedToMempool(ptx); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, + ptx->GetHash().ToString(), + ptx->GetWitnessHash().ToString()); +} + +void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef &ptx) { + auto event = [ptx, this] { + m_internals->TransactionRemovedFromMempool(ptx); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, + ptx->GetHash().ToString(), + ptx->GetWitnessHash().ToString()); } void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>>& pvtxConflicted) { - m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, pvtxConflicted, this] { + auto event = [pblock, pindex, pvtxConflicted, this] { m_internals->BlockConnected(pblock, pindex, *pvtxConflicted); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, + pblock->GetHash().ToString(), + pindex->nHeight); } -void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { - m_internals->m_schedulerClient.AddToProcessQueue([pblock, this] { - m_internals->BlockDisconnected(pblock); - }); +void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +{ + auto event = [pblock, pindex, this] { + m_internals->BlockDisconnected(pblock, pindex); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, + pblock->GetHash().ToString(), + pindex->nHeight); } void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { - m_internals->m_schedulerClient.AddToProcessQueue([locator, this] { + auto event = [locator, this] { m_internals->ChainStateFlushed(locator); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s", __func__, + locator.IsNull() ? "null" : locator.vHave.front().ToString()); } -void CMainSignals::BlockChecked(const CBlock& block, const CValidationState& state) { +void CMainSignals::BlockChecked(const CBlock& block, const BlockValidationState& state) { + LOG_EVENT("%s: block hash=%s state=%s", __func__, + block.GetHash().ToString(), FormatStateMessage(state)); m_internals->BlockChecked(block, state); } void CMainSignals::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock> &block) { + LOG_EVENT("%s: block hash=%s", __func__, block->GetHash().ToString()); m_internals->NewPoWValidBlock(pindex, block); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 3ce617b827..ed6c560944 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,17 +12,15 @@ #include <functional> #include <memory> -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; +class BlockValidationState; class CBlock; class CBlockIndex; struct CBlockLocator; class CConnman; class CValidationInterface; -class CValidationState; class uint256; class CScheduler; -class CTxMemPool; -enum class MemPoolRemovalReason; // These functions dispatch to one or all registered wallets @@ -114,7 +112,7 @@ protected: * * Called on a background thread. */ - virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) {} /** * Notifies listeners of the new active block chain on-disk. * @@ -134,11 +132,11 @@ protected: virtual void ChainStateFlushed(const CBlockLocator &locator) {} /** * Notifies listeners of a block validation result. - * If the provided CValidationState IsValid, the provided block + * If the provided BlockValidationState IsValid, the provided block * is guaranteed to be the current best block at the time the * callback was generated (not necessarily now) */ - virtual void BlockChecked(const CBlock&, const CValidationState&) {} + virtual void BlockChecked(const CBlock&, const BlockValidationState&) {} /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ @@ -158,8 +156,6 @@ private: friend void ::UnregisterAllValidationInterfaces(); friend void ::CallFunctionInValidationInterfaceQueue(std::function<void ()> func); - void MempoolEntryRemoved(CTransactionRef tx, MemPoolRemovalReason reason); - public: /** Register a CScheduler to give callbacks which should run in the background (may only be called once) */ void RegisterBackgroundSignalScheduler(CScheduler& scheduler); @@ -170,17 +166,14 @@ public: size_t CallbacksPending(); - /** Register with mempool to call TransactionRemovedFromMempool callbacks */ - void RegisterWithMempoolSignals(CTxMemPool& pool); - /** Unregister with mempool */ - void UnregisterWithMempoolSignals(CTxMemPool& pool); void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &); + void TransactionRemovedFromMempool(const CTransactionRef &); void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); - void BlockDisconnected(const std::shared_ptr<const CBlock> &); + void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); void ChainStateFlushed(const CBlockLocator &); - void BlockChecked(const CBlock&, const CValidationState&); + void BlockChecked(const CBlock&, const BlockValidationState&); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); }; diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 2285579cd9..af07c67ccf 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/versionbits.h b/src/versionbits.h index d8dda7d95b..b02f848b67 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/versionbitsinfo.cpp b/src/versionbitsinfo.cpp index 82df92ac90..20297b9f9d 100644 --- a/src/versionbitsinfo.cpp +++ b/src/versionbitsinfo.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index 14513bc9e9..c83e598825 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 92a290530c..2893d0ab3d 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,16 +5,19 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include <optional.h> +#include <outputtype.h> #include <policy/feerate.h> #include <policy/fees.h> #include <primitives/transaction.h> -#include <wallet/wallet.h> - -#include <boost/optional.hpp> +#include <script/standard.h> const int DEFAULT_MIN_DEPTH = 0; const int DEFAULT_MAX_DEPTH = 9999999; +//! Default for -avoidpartialspends +static constexpr bool DEFAULT_AVOIDPARTIALSPENDS = false; + /** Coin Control Features. */ class CCoinControl { @@ -22,7 +25,7 @@ public: //! Custom change destination, if not set an address is generated CTxDestination destChange; //! Override the default change type if set, ignored if destChange is set - boost::optional<OutputType> m_change_type; + Optional<OutputType> m_change_type; //! If false, allows unselected inputs, but requires all selected inputs be used bool fAllowOtherInputs; //! Includes watch only addresses which are solvable @@ -30,11 +33,11 @@ public: //! Override automatic min/max checks on fee, m_feerate must be set if true bool fOverrideFeeRate; //! Override the wallet's m_pay_tx_fee if set - boost::optional<CFeeRate> m_feerate; + Optional<CFeeRate> m_feerate; //! Override the default confirmation target if set - boost::optional<unsigned int> m_confirm_target; + Optional<unsigned int> m_confirm_target; //! Override the wallet's m_signal_rbf if set - boost::optional<bool> m_signal_bip125_rbf; + Optional<bool> m_signal_bip125_rbf; //! Avoid partial use of funds sent to a given address bool m_avoid_partial_spends; //! Forbids inclusion of dirty (previously used) addresses diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 8a37f374a1..5bbb2c0ad0 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -1,14 +1,13 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <wallet/coinselection.h> +#include <optional.h> #include <util/system.h> #include <util/moneystr.h> -#include <boost/optional.hpp> - // Descending order comparator struct { bool operator()(const OutputGroup& a, const OutputGroup& b) const @@ -219,7 +218,7 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group nValueRet = 0; // List of values less than target - boost::optional<OutputGroup> lowest_larger; + Optional<OutputGroup> lowest_larger; std::vector<OutputGroup> applicable_groups; CAmount nTotalLower = 0; diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index f6179aa298..b50f00e7d1 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -23,7 +23,7 @@ int CCrypter::BytesToKeySHA512AES(const std::vector<unsigned char>& chSalt, cons unsigned char buf[CSHA512::OUTPUT_SIZE]; CSHA512 di; - di.Write((const unsigned char*)strKeyData.c_str(), strKeyData.size()); + di.Write((const unsigned char*)strKeyData.data(), strKeyData.size()); di.Write(chSalt.data(), chSalt.size()); di.Finalize(buf); diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 4367a5047f..f59c63260e 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index e48eee6c2c..8b042162d8 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -44,7 +44,7 @@ void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filena } } -CCriticalSection cs_db; +RecursiveMutex cs_db; std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. } // namespace @@ -166,10 +166,9 @@ BerkeleyEnvironment::~BerkeleyEnvironment() bool BerkeleyEnvironment::Open(bool retry) { - if (fDbEnvInit) + if (fDbEnvInit) { return true; - - boost::this_thread::interruption_point(); + } fs::path pathIn = strPath; TryCreateDirectories(pathIn); @@ -238,14 +237,12 @@ bool BerkeleyEnvironment::Open(bool retry) return true; } -//! Construct an in-memory mock Berkeley environment for testing and as a place-holder for g_dbenvs emplace +//! Construct an in-memory mock Berkeley environment for testing BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); - boost::this_thread::interruption_point(); - - LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); @@ -263,8 +260,9 @@ BerkeleyEnvironment::BerkeleyEnvironment() DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); - if (ret > 0) + if (ret > 0) { throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); + } fDbEnvInit = true; fMockDb = true; @@ -652,7 +650,7 @@ void BerkeleyEnvironment::ReloadDbEnv() { // Make sure that no Db's are in use AssertLockNotHeld(cs_db); - std::unique_lock<CCriticalSection> lock(cs_db); + std::unique_lock<RecursiveMutex> lock(cs_db); m_db_in_use.wait(lock, [this](){ for (auto& count : mapFileUseCount) { if (count.second > 0) return false; @@ -767,7 +765,7 @@ void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) return; { @@ -776,21 +774,21 @@ void BerkeleyEnvironment::Flush(bool fShutdown) while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); if (!fMockDb) dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else mi++; } - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { @@ -830,7 +828,7 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { - LogPrint(BCLog::DB, "Flushing %s\n", strFile); + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained @@ -838,7 +836,7 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) env->CheckpointLSN(strFile); env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 0a4bb3f396..b93b9ef1bc 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -16,7 +16,7 @@ //! Check whether transaction has descendant in wallet or mempool, or has been //! mined, or conflicts with a mined transaction. Return a feebumper::Result. -static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { if (wallet.HasWalletSpend(wtx.GetHash())) { errors.push_back("Transaction has descendants in the wallet"); @@ -30,7 +30,7 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai } } - if (wtx.GetDepthInMainChain(locked_chain) != 0) { + if (wtx.GetDepthInMainChain() != 0) { errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); return feebumper::Result::WALLET_ERROR; } @@ -47,7 +47,8 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai // check that original tx consists entirely of our inputs // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!wallet.IsAllFromMe(*wtx.tx, ISMINE_SPENDABLE)) { + isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + if (!wallet.IsAllFromMe(*wtx.tx, filter)) { errors.push_back("Transaction contains inputs that don't belong to this wallet"); return feebumper::Result::WALLET_ERROR; } @@ -78,7 +79,8 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt CFeeRate incrementalRelayFee = std::max(wallet.chain().relayIncrementalFee(), CFeeRate(WALLET_INCREMENTAL_RELAY_FEE)); // Given old total fee and transaction size, calculate the old feeRate - CAmount old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + CAmount old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); const int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); CFeeRate nOldFeeRate(old_fee, txSize); // Min total fee is old fee + relay fee @@ -108,12 +110,11 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt return feebumper::Result::OK; } -static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, CCoinControl& coin_control, CAmount& old_fee) +static CFeeRate EstimateFeeRate(const CWallet& wallet, const CWalletTx& wtx, const CAmount old_fee, CCoinControl& coin_control) { // Get the fee rate of the original transaction. This is calculated from // the tx fee/vsize, so it may have been rounded down. Add 1 satoshi to the // result. - old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); CFeeRate feerate(old_fee, txSize); feerate += CFeeRate(1); @@ -146,7 +147,7 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid) if (wtx == nullptr) return false; std::vector<std::string> errors_dummy; - feebumper::Result res = PreconditionChecks(*locked_chain, wallet, *wtx, errors_dummy); + feebumper::Result res = PreconditionChecks(wallet, *wtx, errors_dummy); return res == feebumper::Result::OK; } @@ -165,7 +166,7 @@ Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, co } const CWalletTx& wtx = it->second; - Result result = PreconditionChecks(*locked_chain, *wallet, wtx, errors); + Result result = PreconditionChecks(*wallet, wtx, errors); if (result != Result::OK) { return result; } @@ -196,7 +197,8 @@ Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, co } // calculate the old fee and fee-rate - old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + isminefilter filter = wallet->GetLegacyScriptPubKeyMan() && wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); CFeeRate nOldFeeRate(old_fee, txSize); // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to // future proof against changes to network wide policy for incremental relay @@ -291,7 +293,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } const CWalletTx& wtx = it->second; - Result result = PreconditionChecks(*locked_chain, wallet, wtx, errors); + Result result = PreconditionChecks(wallet, wtx, errors); if (result != Result::OK) { return result; } @@ -309,6 +311,9 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } } + isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); + if (coin_control.m_feerate) { // The user provided a feeRate argument. // We calculate this here to avoid compiler warning on the cs_wallet lock @@ -319,7 +324,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } } else { // The user did not provide a feeRate argument - new_coin_control.m_feerate = EstimateFeeRate(wallet, wtx, new_coin_control, old_fee); + new_coin_control.m_feerate = EstimateFeeRate(wallet, wtx, old_fee, new_coin_control); } // Fill in required inputs we are double-spending(all of them) @@ -382,7 +387,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti CWalletTx& oldWtx = it->second; // make sure the transaction still has no descendants and hasn't been mined in the meantime - Result result = PreconditionChecks(*locked_chain, wallet, oldWtx, errors); + Result result = PreconditionChecks(wallet, oldWtx, errors); if (result != Result::OK) { return result; } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 3657a157b6..2ebc9aba39 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -6,10 +6,13 @@ #include <init.h> #include <interfaces/chain.h> #include <net.h> +#include <node/context.h> #include <outputtype.h> +#include <ui_interface.h> #include <util/moneystr.h> #include <util/system.h> #include <util/translation.h> +#include <wallet/coincontrol.h> #include <wallet/wallet.h> #include <walletinitinterface.h> @@ -25,8 +28,8 @@ public: //! Wallets parameter interaction bool ParameterInteraction() const override; - //! Add wallets that should be opened to list of init interfaces. - void Construct(InitInterfaces& interfaces) const override; + //! Add wallets that should be opened to list of chain clients. + void Construct(NodeContext& node) const override; }; const WalletInitInterface& g_wallet_init_interface = WalletInit(); @@ -43,7 +46,7 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data. 0 to entirely disable the fallbackfee feature. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u). Warning: Smaller sizes may increase the risk of losing funds when restoring from an old backup, if none of the addresses in the original keypool have been used.", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", @@ -57,7 +60,7 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); gArgs.AddArg("-walletbroadcast", strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); #if HAVE_SYSTEM gArgs.AddArg("-walletnotify=<cmd>", "Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #endif @@ -125,12 +128,12 @@ bool WalletInit::ParameterInteraction() const return true; } -void WalletInit::Construct(InitInterfaces& interfaces) const +void WalletInit::Construct(NodeContext& node) const { if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { LogPrintf("Wallet disabled!\n"); return; } gArgs.SoftSetArg("-wallet", ""); - interfaces.chain_clients.emplace_back(interfaces::MakeWalletClient(*interfaces.chain, gArgs.GetArgs("-wallet"))); + node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, gArgs.GetArgs("-wallet"))); } diff --git a/src/wallet/ismine.cpp b/src/wallet/ismine.cpp deleted file mode 100644 index 029b922785..0000000000 --- a/src/wallet/ismine.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <wallet/ismine.h> - -#include <key.h> -#include <script/script.h> -#include <script/signingprovider.h> -#include <wallet/wallet.h> - -typedef std::vector<unsigned char> valtype; - -namespace { - -/** - * This is an enum that tracks the execution context of a script, similar to - * SigVersion in script/interpreter. It is separate however because we want to - * distinguish between top-level scriptPubKey execution and P2SH redeemScript - * execution (a distinction that has no impact on consensus rules). - */ -enum class IsMineSigVersion -{ - TOP = 0, //!< scriptPubKey execution - P2SH = 1, //!< P2SH redeemScript - WITNESS_V0 = 2, //!< P2WSH witness script execution -}; - -/** - * This is an internal representation of isminetype + invalidity. - * Its order is significant, as we return the max of all explored - * possibilities. - */ -enum class IsMineResult -{ - NO = 0, //!< Not ours - WATCH_ONLY = 1, //!< Included in watch-only balance - SPENDABLE = 2, //!< Included in all balances - INVALID = 3, //!< Not spendable by anyone (uncompressed pubkey in segwit, P2SH inside P2SH or witness, witness inside witness) -}; - -bool PermitsUncompressed(IsMineSigVersion sigversion) -{ - return sigversion == IsMineSigVersion::TOP || sigversion == IsMineSigVersion::P2SH; -} - -bool HaveKeys(const std::vector<valtype>& pubkeys, const CWallet& keystore) -{ - for (const valtype& pubkey : pubkeys) { - CKeyID keyID = CPubKey(pubkey).GetID(); - if (!keystore.HaveKey(keyID)) return false; - } - return true; -} - -IsMineResult IsMineInner(const CWallet& keystore, const CScript& scriptPubKey, IsMineSigVersion sigversion) -{ - IsMineResult ret = IsMineResult::NO; - - std::vector<valtype> vSolutions; - txnouttype whichType = Solver(scriptPubKey, vSolutions); - - CKeyID keyID; - switch (whichType) - { - case TX_NONSTANDARD: - case TX_NULL_DATA: - case TX_WITNESS_UNKNOWN: - break; - case TX_PUBKEY: - keyID = CPubKey(vSolutions[0]).GetID(); - if (!PermitsUncompressed(sigversion) && vSolutions[0].size() != 33) { - return IsMineResult::INVALID; - } - if (keystore.HaveKey(keyID)) { - ret = std::max(ret, IsMineResult::SPENDABLE); - } - break; - case TX_WITNESS_V0_KEYHASH: - { - if (sigversion == IsMineSigVersion::WITNESS_V0) { - // P2WPKH inside P2WSH is invalid. - return IsMineResult::INVALID; - } - if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { - // We do not support bare witness outputs unless the P2SH version of it would be - // acceptable as well. This protects against matching before segwit activates. - // This also applies to the P2WSH case. - break; - } - ret = std::max(ret, IsMineInner(keystore, GetScriptForDestination(PKHash(uint160(vSolutions[0]))), IsMineSigVersion::WITNESS_V0)); - break; - } - case TX_PUBKEYHASH: - keyID = CKeyID(uint160(vSolutions[0])); - if (!PermitsUncompressed(sigversion)) { - CPubKey pubkey; - if (keystore.GetPubKey(keyID, pubkey) && !pubkey.IsCompressed()) { - return IsMineResult::INVALID; - } - } - if (keystore.HaveKey(keyID)) { - ret = std::max(ret, IsMineResult::SPENDABLE); - } - break; - case TX_SCRIPTHASH: - { - if (sigversion != IsMineSigVersion::TOP) { - // P2SH inside P2WSH or P2SH is invalid. - return IsMineResult::INVALID; - } - CScriptID scriptID = CScriptID(uint160(vSolutions[0])); - CScript subscript; - if (keystore.GetCScript(scriptID, subscript)) { - ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::P2SH)); - } - break; - } - case TX_WITNESS_V0_SCRIPTHASH: - { - if (sigversion == IsMineSigVersion::WITNESS_V0) { - // P2WSH inside P2WSH is invalid. - return IsMineResult::INVALID; - } - if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { - break; - } - uint160 hash; - CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(hash.begin()); - CScriptID scriptID = CScriptID(hash); - CScript subscript; - if (keystore.GetCScript(scriptID, subscript)) { - ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::WITNESS_V0)); - } - break; - } - - case TX_MULTISIG: - { - // Never treat bare multisig outputs as ours (they can still be made watchonly-though) - if (sigversion == IsMineSigVersion::TOP) { - break; - } - - // Only consider transactions "mine" if we own ALL the - // keys involved. Multi-signature transactions that are - // partially owned (somebody else has a key that can spend - // them) enable spend-out-from-under-you attacks, especially - // in shared-wallet situations. - std::vector<valtype> keys(vSolutions.begin()+1, vSolutions.begin()+vSolutions.size()-1); - if (!PermitsUncompressed(sigversion)) { - for (size_t i = 0; i < keys.size(); i++) { - if (keys[i].size() != 33) { - return IsMineResult::INVALID; - } - } - } - if (HaveKeys(keys, keystore)) { - ret = std::max(ret, IsMineResult::SPENDABLE); - } - break; - } - } - - if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { - ret = std::max(ret, IsMineResult::WATCH_ONLY); - } - return ret; -} - -} // namespace - -isminetype IsMine(const CWallet& keystore, const CScript& scriptPubKey) -{ - switch (IsMineInner(keystore, scriptPubKey, IsMineSigVersion::TOP)) { - case IsMineResult::INVALID: - case IsMineResult::NO: - return ISMINE_NO; - case IsMineResult::WATCH_ONLY: - return ISMINE_WATCH_ONLY; - case IsMineResult::SPENDABLE: - return ISMINE_SPENDABLE; - } - assert(false); -} - -isminetype IsMine(const CWallet& keystore, const CTxDestination& dest) -{ - CScript script = GetScriptForDestination(dest); - return IsMine(keystore, script); -} diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 41555fcb93..5cdd7dff80 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,9 +28,6 @@ enum isminetype : unsigned int /** used for bitflags of isminetype */ typedef uint8_t isminefilter; -isminetype IsMine(const CWallet& wallet, const CScript& scriptPubKey); -isminetype IsMine(const CWallet& wallet, const CTxDestination& dest); - /** * Cachable amount subdivided into watchonly and spendable parts. */ diff --git a/src/wallet/psbtwallet.cpp b/src/wallet/psbtwallet.cpp index 721a244afb..d995fb06d4 100644 --- a/src/wallet/psbtwallet.cpp +++ b/src/wallet/psbtwallet.cpp @@ -39,12 +39,38 @@ TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& ps return TransactionError::SIGHASH_MISMATCH; } - complete &= SignPSBTInput(HidingSigningProvider(pwallet, !sign, !bip32derivs), psbtx, i, sighash_type); + // Get the scriptPubKey to know which SigningProvider to use + CScript script; + if (!input.witness_utxo.IsNull()) { + script = input.witness_utxo.scriptPubKey; + } else if (input.non_witness_utxo) { + if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { + return TransactionError::MISSING_INPUTS; + } + script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; + } else { + // There's no UTXO so we can just skip this now + complete = false; + continue; + } + SignatureData sigdata; + input.FillSignatureData(sigdata); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(script, sigdata); + if (!provider) { + complete = false; + continue; + } + + complete &= SignPSBTInput(HidingSigningProvider(provider.get(), !sign, !bip32derivs), psbtx, i, sighash_type); } // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { - UpdatePSBTOutput(HidingSigningProvider(pwallet, true, !bip32derivs), psbtx, i); + const CTxOut& out = psbtx.tx->vout.at(i); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(out.scriptPubKey); + if (provider) { + UpdatePSBTOutput(HidingSigningProvider(provider.get(), true, !bip32derivs), psbtx, i); + } } return TransactionError::OK; diff --git a/src/wallet/psbtwallet.h b/src/wallet/psbtwallet.h index a7e52df6d9..b35a0a58d1 100644 --- a/src/wallet/psbtwallet.h +++ b/src/wallet/psbtwallet.h @@ -15,8 +15,8 @@ * finalize.) Sets `error` and returns false if something goes wrong. * * @param[in] pwallet pointer to a wallet - * @param[in] &psbtx reference to PartiallySignedTransaction to fill in - * @param[out] &complete indicates whether the PSBT is now complete + * @param[in] psbtx PartiallySignedTransaction to fill in + * @param[out] complete indicates whether the PSBT is now complete * @param[in] sighash_type the sighash type to use when signing (if PSBT does not specify) * @param[in] sign whether to sign or not * @param[in] bip32derivs whether to fill in bip32 derivation information if available diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 1cd4cb93b4..b730d4a4dd 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -23,23 +23,10 @@ #include <tuple> #include <boost/algorithm/string.hpp> -#include <boost/date_time/posix_time/posix_time.hpp> #include <univalue.h> -int64_t static DecodeDumpTime(const std::string &str) { - static const boost::posix_time::ptime epoch = boost::posix_time::from_time_t(0); - static const std::locale loc(std::locale::classic(), - new boost::posix_time::time_input_facet("%Y-%m-%dT%H:%M:%SZ")); - std::istringstream iss(str); - iss.imbue(loc); - boost::posix_time::ptime ptime(boost::date_time::not_a_date_time); - iss >> ptime; - if (ptime.is_not_a_date_time()) - return 0; - return (ptime - epoch).total_seconds(); -} std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; @@ -67,11 +54,11 @@ static std::string DecodeDumpString(const std::string &str) { return ret.str(); } -static bool GetWalletAddressesForKey(CWallet* const pwallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static bool GetWalletAddressesForKey(LegacyScriptPubKeyMan* spk_man, CWallet* const pwallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { bool fLabelFound = false; CKey key; - pwallet->GetKey(keyid, key); + spk_man->GetKey(keyid, key); for (const auto& dest : GetAllDestinationsForKey(key.GetPubKey())) { if (pwallet->mapAddressBook.count(dest)) { if (!strAddr.empty()) { @@ -138,6 +125,8 @@ UniValue importprivkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } + EnsureLegacyScriptPubKeyMan(*wallet, true); + WalletRescanReserver reserver(pwallet); bool fRescan = true; { @@ -170,7 +159,7 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!key.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); + CHECK_NONFATAL(key.VerifyPubKey(pubkey)); CKeyID vchAddress = pubkey.GetID(); { pwallet->MarkDirty(); @@ -264,6 +253,7 @@ UniValue importaddress(const JSONRPCRequest& request) }, }.Check(request); + EnsureLegacyScriptPubKeyMan(*pwallet, true); std::string strLabel; if (!request.params[1].isNull()) @@ -326,7 +316,7 @@ UniValue importaddress(const JSONRPCRequest& request) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } } @@ -364,28 +354,26 @@ UniValue importprunedfunds(const JSONRPCRequest& request) //Search partial merkle tree in proof for our transaction and index in valid block std::vector<uint256> vMatch; std::vector<unsigned int> vIndex; - unsigned int txnIndex = 0; - if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) == merkleBlock.header.hashMerkleRoot) { - - auto locked_chain = pwallet->chain().lock(); - if (locked_chain->getBlockHeight(merkleBlock.header.GetHash()) == nullopt) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); - } - - std::vector<uint256>::const_iterator it; - if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx))==vMatch.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); - } + if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); + } - txnIndex = vIndex[it - vMatch.begin()]; + auto locked_chain = pwallet->chain().lock(); + Optional<int> height = locked_chain->getBlockHeight(merkleBlock.header.GetHash()); + if (height == nullopt) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } - else { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); + + std::vector<uint256>::const_iterator it; + if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx)) == vMatch.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); } - wtx.SetConf(CWalletTx::Status::CONFIRMED, merkleBlock.header.GetHash(), txnIndex); + unsigned int txnIndex = vIndex[it - vMatch.begin()]; + + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *height, merkleBlock.header.GetHash(), txnIndex); + wtx.m_confirm = confirm; - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (pwallet->IsMine(*wtx.tx)) { @@ -466,6 +454,7 @@ UniValue importpubkey(const JSONRPCRequest& request) }, }.Check(request); + EnsureLegacyScriptPubKeyMan(*wallet, true); std::string strLabel; if (!request.params[1].isNull()) @@ -516,7 +505,7 @@ UniValue importpubkey(const JSONRPCRequest& request) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } } @@ -549,6 +538,8 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.Check(request); + EnsureLegacyScriptPubKeyMan(*wallet, true); + if (pwallet->chain().havePruned()) { // Exit early and print an error. // If a block is pruned after this check, we will import the key(s), @@ -598,7 +589,7 @@ UniValue importwallet(const JSONRPCRequest& request) continue; CKey key = DecodeSecret(vstr[0]); if (key.IsValid()) { - int64_t nTime = DecodeDumpTime(vstr[1]); + int64_t nTime = ParseISO8601DateTime(vstr[1]); std::string strLabel; bool fLabel = true; for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { @@ -617,7 +608,7 @@ UniValue importwallet(const JSONRPCRequest& request) } else if(IsHex(vstr[0])) { std::vector<unsigned char> vData(ParseHex(vstr[0])); CScript script = CScript(vData.begin(), vData.end()); - int64_t birth_time = DecodeDumpTime(vstr[1]); + int64_t birth_time = ParseISO8601DateTime(vstr[1]); scripts.push_back(std::pair<CScript, int64_t>(script, birth_time)); } } @@ -637,7 +628,7 @@ UniValue importwallet(const JSONRPCRequest& request) std::string label = std::get<3>(key_tuple); CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); + CHECK_NONFATAL(key.VerifyPubKey(pubkey)); CKeyID keyid = pubkey.GetID(); pwallet->WalletLogPrintf("Importing %s...\n", EncodeDestination(PKHash(keyid))); @@ -706,8 +697,10 @@ UniValue dumpprivkey(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); + auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -716,12 +709,12 @@ UniValue dumpprivkey(const JSONRPCRequest& request) if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } - auto keyid = GetKeyForDestination(*pwallet, dest); + auto keyid = GetKeyForDestination(spk_man, dest); if (keyid.IsNull()) { throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to a key"); } CKey vchSecret; - if (!pwallet->GetKey(keyid, vchSecret)) { + if (!spk_man.GetKey(keyid, vchSecret)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key for address " + strAddress + " is not known"); } return EncodeSecret(vchSecret); @@ -755,8 +748,10 @@ UniValue dumpwallet(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); + auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -778,10 +773,10 @@ UniValue dumpwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); std::map<CKeyID, int64_t> mapKeyBirth; - const std::map<CKeyID, int64_t>& mapKeyPool = pwallet->GetAllReserveKeys(); + const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); pwallet->GetKeyBirthTimes(*locked_chain, mapKeyBirth); - std::set<CScriptID> scripts = pwallet->GetCScripts(); + std::set<CScriptID> scripts = spk_man.GetCScripts(); // sort time/key pairs std::vector<std::pair<int64_t, CKeyID> > vKeyBirth; @@ -800,11 +795,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) file << "\n"; // add the base58check encoded extended master if the wallet uses HD - CKeyID seed_id = pwallet->GetHDChain().seed_id; + CKeyID seed_id = spk_man.GetHDChain().seed_id; if (!seed_id.IsNull()) { CKey seed; - if (pwallet->GetKey(seed_id, seed)) { + if (spk_man.GetKey(seed_id, seed)) { CExtKey masterKey; masterKey.SetSeed(seed.begin(), seed.size()); @@ -817,20 +812,20 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::string strAddr; std::string strLabel; CKey key; - if (pwallet->GetKey(keyid, key)) { + if (spk_man.GetKey(keyid, key)) { file << strprintf("%s %s ", EncodeSecret(key), strTime); - if (GetWalletAddressesForKey(pwallet, keyid, strAddr, strLabel)) { + if (GetWalletAddressesForKey(&spk_man, pwallet, keyid, strAddr, strLabel)) { file << strprintf("label=%s", strLabel); } else if (keyid == seed_id) { file << "hdseed=1"; } else if (mapKeyPool.count(keyid)) { file << "reserve=1"; - } else if (pwallet->mapKeyMetadata[keyid].hdKeypath == "s") { + } else if (spk_man.mapKeyMetadata[keyid].hdKeypath == "s") { file << "inactivehdseed=1"; } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(pwallet->mapKeyMetadata[keyid].key_origin.path) : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (spk_man.mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(spk_man.mapKeyMetadata[keyid].key_origin.path) : "")); } } file << "\n"; @@ -839,11 +834,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::string create_time = "0"; std::string address = EncodeDestination(ScriptHash(scriptid)); // get birth times for scripts with metadata - auto it = pwallet->m_script_metadata.find(scriptid); - if (it != pwallet->m_script_metadata.end()) { + auto it = spk_man.m_script_metadata.find(scriptid); + if (it != spk_man.m_script_metadata.end()) { create_time = FormatISO8601DateTime(it->second.nCreateTime); } - if(pwallet->GetCScript(scriptid, script)) { + if(spk_man.GetCScript(scriptid, script)) { file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); file << strprintf(" # addr=%s\n", address); } @@ -899,7 +894,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TX_SCRIPTHASH: { if (script_ctx == ScriptContext::P2SH) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside another P2SH"); if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside a P2WSH"); - assert(script_ctx == ScriptContext::TOP); + CHECK_NONFATAL(script_ctx == ScriptContext::TOP); CScriptID id = CScriptID(uint160(solverdata[0])); auto subscript = std::move(import_data.redeemscript); // Remove redeemscript from import_data to check for superfluous script later. if (!subscript) return "missing redeemscript"; @@ -1219,7 +1214,7 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con // Check whether we have any work to do for (const CScript& script : script_pub_keys) { - if (::IsMine(*pwallet, script) & ISMINE_SPENDABLE) { + if (pwallet->IsMine(script) & ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script.begin(), script.end()) + "\")"); } } @@ -1290,7 +1285,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) {"scriptPubKey", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of scriptPubKey (string for script, json for address). Should not be provided if using a descriptor", /* oneline_description */ "", {"\"<script>\" | { \"address\":\"<address>\" }", "string / json"} }, - {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Creation time of the key in seconds since epoch (Jan 1 1970 GMT),\n" + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Creation time of the key expressed in " + UNIX_EPOCH_TIME + ",\n" " or the string \"now\" to substitute the current synced blockchain time. The timestamp of the oldest\n" " key will determine how far back blockchain rescans need to begin for missing wallet transactions.\n" " \"now\" can be specified to bypass scanning, for keys which are known to never have been used, and\n" @@ -1339,6 +1334,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); + EnsureLegacyScriptPubKeyMan(*wallet, true); + const UniValue& requests = mainRequest.params[0]; //Default options @@ -1406,7 +1403,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } if (pwallet->IsAbortingRescan()) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e571501221..ab0819a821 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <amount.h> #include <core_io.h> -#include <init.h> #include <interfaces/chain.h> #include <key_io.h> +#include <node/context.h> #include <outputtype.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -68,7 +68,7 @@ static bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWall /** Checks if a CKey is in the given CWallet compressed or otherwise*/ -bool HaveKey(const CWallet& wallet, const CKey& key) +bool HaveKey(const SigningProvider& wallet, const CKey& key) { CKey key2; key2.Set(key.begin(), key.end(), !key.IsCompressed()); @@ -124,19 +124,33 @@ void EnsureWalletIsUnlocked(const CWallet* pwallet) } } +// also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create) +{ + LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man && also_create) { + spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + } + if (!spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); + } + return *spk_man; +} + static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, UniValue& entry) { - int confirms = wtx.GetDepthInMainChain(locked_chain); + int confirms = wtx.GetDepthInMainChain(); entry.pushKV("confirmations", confirms); if (wtx.IsCoinBase()) entry.pushKV("generated", true); if (confirms > 0) { entry.pushKV("blockhash", wtx.m_confirm.hashBlock.GetHex()); + entry.pushKV("blockheight", wtx.m_confirm.block_height); entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); - assert(found_block); + CHECK_NONFATAL(found_block); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); @@ -303,7 +317,7 @@ static UniValue setlabel(const JSONRPCRequest& request) std::string label = LabelFromValue(request.params[1]); - if (IsMine(*pwallet, dest)) { + if (pwallet->IsMine(dest)) { pwallet->SetAddressBook(dest, label, "receive"); } else { pwallet->SetAddressBook(dest, label, "send"); @@ -550,9 +564,15 @@ static UniValue signmessage(const JSONRPCRequest& request) throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); } + CScript script_pub_key = GetScriptForDestination(*pkhash); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(script_pub_key); + if (!provider) { + throw JSONRPCError(RPC_WALLET_ERROR, "Private key not available"); + } + CKey key; CKeyID keyID(*pkhash); - if (!pwallet->GetKey(keyID, key)) { + if (!provider->GetKey(keyID, key)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key not available"); } @@ -610,7 +630,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } CScript scriptPubKey = GetScriptForDestination(dest); - if (!IsMine(*pwallet, scriptPubKey)) { + if (!pwallet->IsMine(scriptPubKey)) { throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); } @@ -629,7 +649,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) for (const CTxOut& txout : wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain(*locked_chain) >= nMinDepth) + if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } @@ -694,8 +714,8 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) for (const CTxOut& txout : wtx.tx->vout) { CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && IsMine(*pwallet, address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain(*locked_chain) >= nMinDepth) + if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { + if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } } @@ -935,7 +955,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) } RPCHelpMan{"addmultisigaddress", - "\nAdd a nrequired-to-sign multisignature address to the wallet. Requires a new wallet backup.\n" + "\nAdd an nrequired-to-sign multisignature address to the wallet. Requires a new wallet backup.\n" "Each key is a Bitcoin address or hex-encoded public key.\n" "This functionality is only intended for use with non-watchonly addresses.\n" "See `importaddress` for watchonly p2sh address support.\n" @@ -964,8 +984,10 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); std::string label; if (!request.params[2].isNull()) @@ -980,7 +1002,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) if (IsHex(keys_or_addrs[i].get_str()) && (keys_or_addrs[i].get_str().length() == 66 || keys_or_addrs[i].get_str().length() == 130)) { pubkeys.push_back(HexToPubKey(keys_or_addrs[i].get_str())); } else { - pubkeys.push_back(AddrToPubKey(pwallet, keys_or_addrs[i].get_str())); + pubkeys.push_back(AddrToPubKey(spk_man, keys_or_addrs[i].get_str())); } } @@ -993,7 +1015,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) // Construct using pay-to-script-hash: CScript inner; - CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, *pwallet, inner); + CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, spk_man, inner); pwallet->SetAddressBook(dest, label, "send"); UniValue result(UniValue::VOBJ); @@ -1050,7 +1072,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co continue; } - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < nMinDepth) continue; @@ -1064,7 +1086,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co continue; } - isminefilter mine = IsMine(*pwallet, address); + isminefilter mine = pwallet->IsMine(address); if(!(mine & filter)) continue; @@ -1288,7 +1310,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con for (const COutputEntry& s : listSent) { UniValue entry(UniValue::VOBJ); - if (involvesWatchonly || (::IsMine(*pwallet, s.destination) & ISMINE_WATCH_ONLY)) { + if (involvesWatchonly || (pwallet->IsMine(s.destination) & ISMINE_WATCH_ONLY)) { entry.pushKV("involvesWatchonly", true); } MaybePushAddress(entry, s.destination); @@ -1307,8 +1329,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con } // Received - if (listReceived.size() > 0 && wtx.GetDepthInMainChain(locked_chain) >= nMinDepth) - { + if (listReceived.size() > 0 && wtx.GetDepthInMainChain() >= nMinDepth) { for (const COutputEntry& r : listReceived) { std::string label; @@ -1319,15 +1340,15 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con continue; } UniValue entry(UniValue::VOBJ); - if (involvesWatchonly || (::IsMine(*pwallet, r.destination) & ISMINE_WATCH_ONLY)) { + if (involvesWatchonly || (pwallet->IsMine(r.destination) & ISMINE_WATCH_ONLY)) { entry.pushKV("involvesWatchonly", true); } MaybePushAddress(entry, r.destination); if (wtx.IsCoinBase()) { - if (wtx.GetDepthInMainChain(locked_chain) < 1) + if (wtx.GetDepthInMainChain() < 1) entry.pushKV("category", "orphan"); - else if (wtx.IsImmatureCoinBase(locked_chain)) + else if (wtx.IsImmatureCoinBase()) entry.pushKV("category", "immature"); else entry.pushKV("category", "generate"); @@ -1355,15 +1376,16 @@ static const std::string TransactionDescriptionString() " \"generated\": xxx, (bool) Only present if transaction only input is a coinbase one.\n" " \"trusted\": xxx, (bool) Only present if we consider transaction to be trusted and so safe to spend from.\n" " \"blockhash\": \"hashvalue\", (string) The block hash containing the transaction.\n" + " \"blockheight\": n, (numeric) The block height containing the transaction.\n" " \"blockindex\": n, (numeric) The index of the transaction in the block that includes it.\n" - " \"blocktime\": xxx, (numeric) The block time in seconds since epoch (1 Jan 1970 GMT).\n" + " \"blocktime\": xxx, (numeric) The block time expressed in " + UNIX_EPOCH_TIME + ".\n" " \"txid\": \"transactionid\", (string) The transaction id.\n" " \"walletconflicts\": [ (array) Conflicting transaction ids.\n" " \"txid\", (string) The transaction id.\n" " ...\n" " ],\n" - " \"time\": xxx, (numeric) The transaction time in seconds since epoch (midnight Jan 1 1970 GMT).\n" - " \"timereceived\": xxx, (numeric) The time received in seconds since epoch (midnight Jan 1 1970 GMT).\n" + " \"time\": xxx, (numeric) The transaction time expressed in " + UNIX_EPOCH_TIME + ".\n" + " \"timereceived\": xxx, (numeric) The time received expressed in " + UNIX_EPOCH_TIME + ".\n" " \"comment\": \"...\", (string) If a comment is associated with the transaction, only present if not empty.\n" " \"bip125-replaceable\": \"yes|no|unknown\", (string) Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" " may be unknown for unconfirmed transactions not in the mempool\n"; @@ -1591,7 +1613,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; - if (depth == -1 || tx.GetDepthInMainChain(*locked_chain) < depth) { + if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1708,7 +1730,7 @@ static UniValue gettransaction(const JSONRPCRequest& request) } const CWalletTx& wtx = it->second; - CAmount nCredit = wtx.GetCredit(*locked_chain, filter); + CAmount nCredit = wtx.GetCredit(filter); CAmount nDebit = wtx.GetDebit(filter); CAmount nNet = nCredit - nDebit; CAmount nFee = (wtx.IsFromMe(filter) ? wtx.tx->GetValueOut() - nDebit : 0); @@ -1772,7 +1794,7 @@ static UniValue abandontransaction(const JSONRPCRequest& request) if (!pwallet->mapWallet.count(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); } - if (!pwallet->AbandonTransaction(*locked_chain, hash)) { + if (!pwallet->AbandonTransaction(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not eligible for abandonment"); } @@ -2203,7 +2225,7 @@ static UniValue lockunspent(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout index out of bounds"); } - if (pwallet->IsSpent(*locked_chain, outpt.hash, outpt.n)) { + if (pwallet->IsSpent(outpt.hash, outpt.n)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected unspent output"); } @@ -2379,7 +2401,8 @@ static UniValue getbalances(const JSONRPCRequest& request) } balances.pushKV("mine", balances_mine); } - if (wallet.HaveWatchOnly()) { + auto spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (spk_man && spk_man->HaveWatchOnly()) { UniValue balances_watchonly{UniValue::VOBJ}; balances_watchonly.pushKV("trusted", ValueFromAmount(bal.m_watchonly_trusted)); balances_watchonly.pushKV("untrusted_pending", ValueFromAmount(bal.m_watchonly_untrusted_pending)); @@ -2409,10 +2432,10 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) " \"unconfirmed_balance\": xxx, (numeric) DEPRECATED. Identical to getbalances().mine.untrusted_pending\n" " \"immature_balance\": xxxxxx, (numeric) DEPRECATED. Identical to getbalances().mine.immature\n" " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoololdest\": xxxxxx, (numeric) the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool\n" " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated (only counts external keys)\n" " \"keypoolsize_hd_internal\": xxxx, (numeric) how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)\n" - " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" + " \"unlocked_until\": ttt, (numeric) the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" " \"hdseedid\": \"<hash160>\" (string, optional) the Hash160 of the HD seed (only present when HD is enabled)\n" " \"private_keys_enabled\": true|false (boolean) false if privatekeys are disabled for this wallet (enforced watch-only wallet)\n" @@ -2449,7 +2472,15 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("txcount", (int)pwallet->mapWallet.size()); obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); obj.pushKV("keypoolsize", (int64_t)kpExternalSize); - CKeyID seed_id = pwallet->GetHDChain().seed_id; + + LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); + if (spk_man) { + CKeyID seed_id = spk_man->GetHDChain().seed_id; + if (!seed_id.IsNull()) { + obj.pushKV("hdseedid", seed_id.GetHex()); + } + } + if (pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { obj.pushKV("keypoolsize_hd_internal", (int64_t)(pwallet->GetKeyPoolSize() - kpExternalSize)); } @@ -2457,9 +2488,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("unlocked_until", pwallet->nRelockTime); } obj.pushKV("paytxfee", ValueFromAmount(pwallet->m_pay_tx_fee.GetFeePerK())); - if (!seed_id.IsNull()) { - obj.pushKV("hdseedid", seed_id.GetHex()); - } obj.pushKV("private_keys_enabled", !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); obj.pushKV("avoid_reuse", pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)); if (pwallet->IsScanning()) { @@ -2574,7 +2602,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) std::string error; std::vector<std::string> warning; - std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_interfaces->chain, location, error, warning); + std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warning); if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error); UniValue obj(UniValue::VOBJ); @@ -2700,7 +2728,7 @@ static UniValue createwallet(const JSONRPCRequest& request) std::string error; std::shared_ptr<CWallet> wallet; - WalletCreationStatus status = CreateWallet(*g_rpc_interfaces->chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); + WalletCreationStatus status = CreateWallet(*g_rpc_chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); switch (status) { case WalletCreationStatus::CREATION_FAILED: throw JSONRPCError(RPC_WALLET_ERROR, error); @@ -2903,7 +2931,7 @@ static UniValue listunspent(const JSONRPCRequest& request) CTxDestination address; const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); - bool reused = avoid_reuse && pwallet->IsUsedDestination(address); + bool reused = avoid_reuse && pwallet->IsUsedDestination(out.tx->GetHash(), out.i); if (destinations.size() && (!fValidAddress || !destinations.count(address))) continue; @@ -2920,33 +2948,36 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("label", i->second.name); } - if (scriptPubKey.IsPayToScriptHash()) { - const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); - CScript redeemScript; - if (pwallet->GetCScript(hash, redeemScript)) { - entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); - // Now check if the redeemScript is actually a P2WSH script - CTxDestination witness_destination; - if (redeemScript.IsPayToWitnessScriptHash()) { - bool extracted = ExtractDestination(redeemScript, witness_destination); - assert(extracted); - // Also return the witness script - const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); - CScriptID id; - CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); - CScript witnessScript; - if (pwallet->GetCScript(id, witnessScript)) { - entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); + if (provider) { + if (scriptPubKey.IsPayToScriptHash()) { + const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); + CScript redeemScript; + if (provider->GetCScript(hash, redeemScript)) { + entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); + // Now check if the redeemScript is actually a P2WSH script + CTxDestination witness_destination; + if (redeemScript.IsPayToWitnessScriptHash()) { + bool extracted = ExtractDestination(redeemScript, witness_destination); + CHECK_NONFATAL(extracted); + // Also return the witness script + const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); + CScriptID id; + CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScript witnessScript; + if (provider->GetCScript(id, witnessScript)) { + entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + } } } - } - } else if (scriptPubKey.IsPayToWitnessScriptHash()) { - const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(address); - CScriptID id; - CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); - CScript witnessScript; - if (pwallet->GetCScript(id, witnessScript)) { - entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + } else if (scriptPubKey.IsPayToWitnessScriptHash()) { + const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(address); + CScriptID id; + CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScript witnessScript; + if (provider->GetCScript(id, witnessScript)) { + entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + } } } } @@ -2957,8 +2988,11 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("spendable", out.fSpendable); entry.pushKV("solvable", out.fSolvable); if (out.fSolvable) { - auto descriptor = InferDescriptor(scriptPubKey, *pwallet); - entry.pushKV("desc", descriptor->ToString()); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); + if (provider) { + auto descriptor = InferDescriptor(scriptPubKey, *provider); + entry.pushKV("desc", descriptor->ToString()); + } } if (avoid_reuse) entry.pushKV("reused", reused); entry.pushKV("safe", out.fSafe); @@ -3267,7 +3301,23 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) // Parse the prevtxs array ParsePrevouts(request.params[1], nullptr, coins); - return SignTransaction(mtx, pwallet, coins, request.params[2]); + std::set<std::shared_ptr<SigningProvider>> providers; + for (const std::pair<COutPoint, Coin> coin_pair : coins) { + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(coin_pair.second.out.scriptPubKey); + if (provider) { + providers.insert(std::move(provider)); + } + } + if (providers.size() == 0) { + // When there are no available providers, use a dummy SigningProvider so we can check if the tx is complete + providers.insert(std::make_shared<SigningProvider>()); + } + + UniValue result(UniValue::VOBJ); + for (std::shared_ptr<SigningProvider> provider : providers) { + SignTransaction(mtx, provider.get(), coins, request.params[2], result); + } + return result; } static UniValue bumpfee(const JSONRPCRequest& request) @@ -3319,10 +3369,11 @@ static UniValue bumpfee(const JSONRPCRequest& request) }, RPCResult{ "{\n" - " \"txid\": \"value\", (string) The id of the new transaction\n" - " \"origfee\": n, (numeric) Fee of the replaced transaction\n" - " \"fee\": n, (numeric) Fee of the new transaction\n" - " \"errors\": [ str... ] (json array of strings) Errors encountered during processing (may be empty)\n" + " \"psbt\": \"psbt\", (string) The base64-encoded unsigned PSBT of the new transaction. Only returned when wallet private keys are disabled.\n" + " \"txid\": \"value\", (string) The id of the new transaction. Only returned when wallet private keys are enabled.\n" + " \"origfee\": n, (numeric) The fee of the replaced transaction.\n" + " \"fee\": n, (numeric) The fee of the new transaction.\n" + " \"errors\": [ str... ] (json array of strings) Errors encountered during processing (may be empty).\n" "}\n" }, RPCExamples{ @@ -3334,10 +3385,12 @@ static UniValue bumpfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); uint256 hash(ParseHashV(request.params[0], "txid")); + CCoinControl coin_control; + coin_control.fAllowWatchOnly = pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); // optional parameters CAmount totalFee = 0; - CCoinControl coin_control; coin_control.m_signal_bip125_rbf = true; + if (!request.params[1].isNull()) { UniValue options = request.params[1]; RPCTypeCheckObj(options, @@ -3422,17 +3475,32 @@ static UniValue bumpfee(const JSONRPCRequest& request) } } - // sign bumped transaction - if (!feebumper::SignTransaction(*pwallet, mtx)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); - } - // commit the bumped transaction - uint256 txid; - if (feebumper::CommitTransaction(*pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { - throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); - } UniValue result(UniValue::VOBJ); - result.pushKV("txid", txid.GetHex()); + + // If wallet private keys are enabled, return the new transaction id, + // otherwise return the base64-encoded unsigned PSBT of the new transaction. + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!feebumper::SignTransaction(*pwallet, mtx)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); + } + + uint256 txid; + if (feebumper::CommitTransaction(*pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { + throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); + } + + result.pushKV("txid", txid.GetHex()); + } else { + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + const TransactionError err = FillPSBT(pwallet, psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + CHECK_NONFATAL(err == TransactionError::OK); + CHECK_NONFATAL(!complete); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + result.pushKV("psbt", EncodeBase64(ssTx.str())); + } + result.pushKV("origfee", ValueFromAmount(old_fee)); result.pushKV("fee", ValueFromAmount(new_fee)); UniValue result_errors(UniValue::VARR); @@ -3539,7 +3607,7 @@ UniValue rescanblockchain(const JSONRPCRequest& request) class DescribeWalletAddressVisitor : public boost::static_visitor<UniValue> { public: - CWallet * const pwallet; + const SigningProvider * const provider; void ProcessSubScript(const CScript& subscript, UniValue& obj) const { @@ -3575,7 +3643,7 @@ public: } } - explicit DescribeWalletAddressVisitor(CWallet* _pwallet) : pwallet(_pwallet) {} + explicit DescribeWalletAddressVisitor(const SigningProvider* _provider) : provider(_provider) {} UniValue operator()(const CNoDestination& dest) const { return UniValue(UniValue::VOBJ); } @@ -3584,7 +3652,7 @@ public: CKeyID keyID(pkhash); UniValue obj(UniValue::VOBJ); CPubKey vchPubKey; - if (pwallet && pwallet->GetPubKey(keyID, vchPubKey)) { + if (provider && provider->GetPubKey(keyID, vchPubKey)) { obj.pushKV("pubkey", HexStr(vchPubKey)); obj.pushKV("iscompressed", vchPubKey.IsCompressed()); } @@ -3596,7 +3664,7 @@ public: CScriptID scriptID(scripthash); UniValue obj(UniValue::VOBJ); CScript subscript; - if (pwallet && pwallet->GetCScript(scriptID, subscript)) { + if (provider && provider->GetCScript(scriptID, subscript)) { ProcessSubScript(subscript, obj); } return obj; @@ -3606,7 +3674,7 @@ public: { UniValue obj(UniValue::VOBJ); CPubKey pubkey; - if (pwallet && pwallet->GetPubKey(CKeyID(id), pubkey)) { + if (provider && provider->GetPubKey(CKeyID(id), pubkey)) { obj.pushKV("pubkey", HexStr(pubkey)); } return obj; @@ -3619,7 +3687,7 @@ public: CRIPEMD160 hasher; uint160 hash; hasher.Write(id.begin(), 32).Finalize(hash.begin()); - if (pwallet && pwallet->GetCScript(CScriptID(hash), subscript)) { + if (provider && provider->GetCScript(CScriptID(hash), subscript)) { ProcessSubScript(subscript, obj); } return obj; @@ -3632,8 +3700,13 @@ static UniValue DescribeWalletAddress(CWallet* pwallet, const CTxDestination& de { UniValue ret(UniValue::VOBJ); UniValue detail = DescribeAddress(dest); + CScript script = GetScriptForDestination(dest); + std::unique_ptr<SigningProvider> provider = nullptr; + if (pwallet) { + provider = pwallet->GetSigningProvider(script); + } ret.pushKVs(detail); - ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(pwallet), dest)); + ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider.get()), dest)); return ret; } @@ -3657,53 +3730,62 @@ UniValue getaddressinfo(const JSONRPCRequest& request) return NullUniValue; } + const std::string example_address = "\"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl\""; + RPCHelpMan{"getaddressinfo", - "\nReturn information about the given bitcoin address. Some information requires the address\n" - "to be in the wallet.\n", + "\nReturn information about the given bitcoin address.\n" + "Some of the information will only be present if the address is in the active wallet.\n", { - {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to get the information of."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address for which to get information."}, }, RPCResult{ "{\n" - " \"address\" : \"address\", (string) The bitcoin address validated\n" - " \"scriptPubKey\" : \"hex\", (string) The hex-encoded scriptPubKey generated by the address\n" - " \"ismine\" : true|false, (boolean) If the address is yours or not\n" - " \"iswatchonly\" : true|false, (boolean) If the address is watchonly\n" - " \"solvable\" : true|false, (boolean) Whether we know how to spend coins sent to this address, ignoring the possible lack of private keys\n" - " \"desc\" : \"desc\", (string, optional) A descriptor for spending coins sent to this address (only when solvable)\n" - " \"isscript\" : true|false, (boolean) If the key is a script\n" - " \"ischange\" : true|false, (boolean) If the address was used for change output\n" - " \"iswitness\" : true|false, (boolean) If the address is a witness address\n" - " \"witness_version\" : version (numeric, optional) The version number of the witness program\n" - " \"witness_program\" : \"hex\" (string, optional) The hex value of the witness program\n" - " \"script\" : \"type\" (string, optional) The output script type. Only if \"isscript\" is true and the redeemscript is known. Possible types: nonstandard, pubkey, pubkeyhash, scripthash, multisig, nulldata, witness_v0_keyhash, witness_v0_scripthash, witness_unknown\n" - " \"hex\" : \"hex\", (string, optional) The redeemscript for the p2sh address\n" - " \"pubkeys\" (string, optional) Array of pubkeys associated with the known redeemscript (only if \"script\" is \"multisig\")\n" + " \"address\" : \"address\", (string) The bitcoin address validated.\n" + " \"scriptPubKey\" : \"hex\", (string) The hex-encoded scriptPubKey generated by the address.\n" + " \"ismine\" : true|false, (boolean) If the address is yours.\n" + " \"iswatchonly\" : true|false, (boolean) If the address is watchonly.\n" + " \"solvable\" : true|false, (boolean) If we know how to spend coins sent to this address, ignoring the possible lack of private keys.\n" + " \"desc\" : \"desc\", (string, optional) A descriptor for spending coins sent to this address (only when solvable).\n" + " \"isscript\" : true|false, (boolean) If the key is a script.\n" + " \"ischange\" : true|false, (boolean) If the address was used for change output.\n" + " \"iswitness\" : true|false, (boolean) If the address is a witness address.\n" + " \"witness_version\" : version (numeric, optional) The version number of the witness program.\n" + " \"witness_program\" : \"hex\" (string, optional) The hex value of the witness program.\n" + " \"script\" : \"type\" (string, optional) The output script type. Only if isscript is true and the redeemscript is known. Possible\n" + " types: nonstandard, pubkey, pubkeyhash, scripthash, multisig, nulldata, witness_v0_keyhash,\n" + " witness_v0_scripthash, witness_unknown.\n" + " \"hex\" : \"hex\", (string, optional) The redeemscript for the p2sh address.\n" + " \"pubkeys\" (array, optional) Array of pubkeys associated with the known redeemscript (only if script is multisig).\n" " [\n" - " \"pubkey\"\n" + " \"pubkey\" (string)\n" " ,...\n" " ]\n" - " \"sigsrequired\" : xxxxx (numeric, optional) Number of signatures required to spend multisig output (only if \"script\" is \"multisig\")\n" - " \"pubkey\" : \"publickeyhex\", (string, optional) The hex value of the raw public key, for single-key addresses (possibly embedded in P2SH or P2WSH)\n" - " \"embedded\" : {...}, (object, optional) Information about the address embedded in P2SH or P2WSH, if relevant and known. It includes all getaddressinfo output fields for the embedded address, excluding metadata (\"timestamp\", \"hdkeypath\", \"hdseedid\") and relation to the wallet (\"ismine\", \"iswatchonly\").\n" - " \"iscompressed\" : true|false, (boolean, optional) If the pubkey is compressed\n" - " \"label\" : \"label\" (string) The label associated with the address, \"\" is the default label\n" - " \"timestamp\" : timestamp, (number, optional) The creation time of the key if available in seconds since epoch (Jan 1 1970 GMT)\n" - " \"hdkeypath\" : \"keypath\" (string, optional) The HD keypath if the key is HD and available\n" - " \"hdseedid\" : \"<hash160>\" (string, optional) The Hash160 of the HD seed\n" - " \"hdmasterfingerprint\" : \"<hash160>\" (string, optional) The fingperint of the master key.\n" - " \"labels\" (object) Array of labels associated with the address.\n" + " \"sigsrequired\" : xxxxx (numeric, optional) The number of signatures required to spend multisig output (only if script is multisig).\n" + " \"pubkey\" : \"publickeyhex\", (string, optional) The hex value of the raw public key for single-key addresses (possibly embedded in P2SH or P2WSH).\n" + " \"embedded\" : {...}, (object, optional) Information about the address embedded in P2SH or P2WSH, if relevant and known. Includes all\n" + " getaddressinfo output fields for the embedded address, excluding metadata (timestamp, hdkeypath,\n" + " hdseedid) and relation to the wallet (ismine, iswatchonly).\n" + " \"iscompressed\" : true|false, (boolean, optional) If the pubkey is compressed.\n" + " \"label\" : \"label\" (string) DEPRECATED. The label associated with the address. Defaults to \"\". Replaced by the labels array below.\n" + " \"timestamp\" : timestamp, (number, optional) The creation time of the key, if available, expressed in " + UNIX_EPOCH_TIME + ".\n" + " \"hdkeypath\" : \"keypath\" (string, optional) The HD keypath, if the key is HD and available.\n" + " \"hdseedid\" : \"<hash160>\" (string, optional) The Hash160 of the HD seed.\n" + " \"hdmasterfingerprint\" : \"<hash160>\" (string, optional) The fingerprint of the master key.\n" + " \"labels\" (array) Array of labels associated with the address. Currently limited to one label but returned\n" + " as an array to keep the API stable if multiple labels are enabled in the future.\n" " [\n" - " { (json object of label data)\n" - " \"name\": \"labelname\" (string) The label\n" - " \"purpose\": \"string\" (string) Purpose of address (\"send\" for sending address, \"receive\" for receiving address)\n" - " },...\n" + " \"label name\" (string) The label name. Defaults to \"\".\n" + " DEPRECATED, will be removed in 0.21. To re-enable, launch bitcoind with `-deprecatedrpc=labelspurpose`:\n" + " {\n" + " \"name\" : \"label name\" (string) The label name. Defaults to \"\".\n" + " \"purpose\" : \"purpose\" (string) The purpose of the associated address (send or receive).\n" + " }\n" " ]\n" "}\n" }, RPCExamples{ - HelpExampleCli("getaddressinfo", "\"1PSSGeFHDnKNxiEyFrD1wcEaHr9hrQDDWc\"") - + HelpExampleRpc("getaddressinfo", "\"1PSSGeFHDnKNxiEyFrD1wcEaHr9hrQDDWc\"") + HelpExampleCli("getaddressinfo", example_address) + + HelpExampleRpc("getaddressinfo", example_address) }, }.Check(request); @@ -3711,7 +3793,6 @@ UniValue getaddressinfo(const JSONRPCRequest& request) UniValue ret(UniValue::VOBJ); CTxDestination dest = DecodeDestination(request.params[0].get_str()); - // Make sure the destination is valid if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); @@ -3723,50 +3804,59 @@ UniValue getaddressinfo(const JSONRPCRequest& request) CScript scriptPubKey = GetScriptForDestination(dest); ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); - isminetype mine = IsMine(*pwallet, dest); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); + + isminetype mine = pwallet->IsMine(dest); ret.pushKV("ismine", bool(mine & ISMINE_SPENDABLE)); - bool solvable = IsSolvable(*pwallet, scriptPubKey); + + bool solvable = provider && IsSolvable(*provider, scriptPubKey); ret.pushKV("solvable", solvable); + if (solvable) { - ret.pushKV("desc", InferDescriptor(scriptPubKey, *pwallet)->ToString()); + ret.pushKV("desc", InferDescriptor(scriptPubKey, *provider)->ToString()); } + ret.pushKV("iswatchonly", bool(mine & ISMINE_WATCH_ONLY)); + UniValue detail = DescribeWalletAddress(pwallet, dest); ret.pushKVs(detail); - if (pwallet->mapAddressBook.count(dest)) { + + // DEPRECATED: Return label field if existing. Currently only one label can + // be associated with an address, so the label should be equivalent to the + // value of the name key/value pair in the labels array below. + if ((pwallet->chain().rpcEnableDeprecated("label")) && (pwallet->mapAddressBook.count(dest))) { ret.pushKV("label", pwallet->mapAddressBook[dest].name); } + ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); - const CKeyMetadata* meta = nullptr; - CKeyID key_id = GetKeyForDestination(*pwallet, dest); - if (!key_id.IsNull()) { - auto it = pwallet->mapKeyMetadata.find(key_id); - if (it != pwallet->mapKeyMetadata.end()) { - meta = &it->second; - } - } - if (!meta) { - auto it = pwallet->m_script_metadata.find(CScriptID(scriptPubKey)); - if (it != pwallet->m_script_metadata.end()) { - meta = &it->second; - } - } - if (meta) { - ret.pushKV("timestamp", meta->nCreateTime); - if (meta->has_key_origin) { - ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); - ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); - ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + + ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); + if (spk_man) { + if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { + ret.pushKV("timestamp", meta->nCreateTime); + if (meta->has_key_origin) { + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); + ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); + ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + } } } - // Currently only one label can be associated with an address, return an array - // so the API remains stable if we allow multiple labels to be associated with - // an address. + // Return a `labels` array containing the label associated with the address, + // equivalent to the `label` field above. Currently only one label can be + // associated with an address, but we return an array so the API remains + // stable if we allow multiple labels to be associated with an address in + // the future. UniValue labels(UniValue::VARR); std::map<CTxDestination, CAddressBookData>::iterator mi = pwallet->mapAddressBook.find(dest); if (mi != pwallet->mapAddressBook.end()) { - labels.push_back(AddressBookDataToJSON(mi->second, true)); + // DEPRECATED: The previous behavior of returning an array containing a + // JSON object of `name` and `purpose` key/value pairs is deprecated. + if (pwallet->chain().rpcEnableDeprecated("labelspurpose")) { + labels.push_back(AddressBookDataToJSON(mi->second, true)); + } else { + labels.push_back(mi->second.name); + } } ret.pushKV("labels", std::move(labels)); @@ -3814,7 +3904,7 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // address strings, but build a separate set as a precaution just in // case it does. bool unique = addresses.emplace(address).second; - assert(unique); + CHECK_NONFATAL(unique); // UniValue::pushKV checks if the key exists in O(N) // and since duplicate addresses are unexpected (checked with // std::set in O(log(N))), UniValue::__pushKV is used instead, @@ -3916,6 +4006,8 @@ UniValue sethdseed(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); + if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); } @@ -3925,7 +4017,7 @@ UniValue sethdseed(const JSONRPCRequest& request) } auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { @@ -3941,22 +4033,22 @@ UniValue sethdseed(const JSONRPCRequest& request) CPubKey master_pub_key; if (request.params[1].isNull()) { - master_pub_key = pwallet->GenerateNewSeed(); + master_pub_key = spk_man.GenerateNewSeed(); } else { CKey key = DecodeSecret(request.params[1].get_str()); if (!key.IsValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); } - if (HaveKey(*pwallet, key)) { + if (HaveKey(spk_man, key)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Already have this key (either as an HD seed or as a loose private key)"); } - master_pub_key = pwallet->DeriveNewSeed(key); + master_pub_key = spk_man.DeriveNewSeed(key); } - pwallet->SetHDSeed(master_pub_key); - if (flush_key_pool) pwallet->NewKeyPool(); + spk_man.SetHDSeed(master_pub_key); + if (flush_key_pool) spk_man.NewKeyPool(); return NullUniValue; } @@ -4231,3 +4323,5 @@ void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) handlers.emplace_back(chain.handleRpc(commands[vcidx])); } + +interfaces::Chain* g_rpc_chain = nullptr; diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 1c0523c90b..2813fa2bfc 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -12,6 +12,7 @@ class CRPCTable; class CWallet; class JSONRPCRequest; +class LegacyScriptPubKeyMan; class UniValue; struct PartiallySignedTransaction; class CTransaction; @@ -21,6 +22,12 @@ class Chain; class Handler; } +//! Pointer to chain interface that needs to be declared as a global to be +//! accessible loadwallet and createwallet methods. Due to limitations of the +//! RPC framework, there's currently no direct way to pass in state to RPC +//! methods without globals. +extern interfaces::Chain* g_rpc_chain; + void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers); /** @@ -34,6 +41,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques std::string HelpRequiringPassphrase(const CWallet*); void EnsureWalletIsUnlocked(const CWallet*); bool EnsureWalletIsAvailable(const CWallet*, bool avoidException); +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); UniValue getaddressinfo(const JSONRPCRequest& request); UniValue signrawtransactionwithwallet(const JSONRPCRequest& request); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp new file mode 100644 index 0000000000..4c9d88973e --- /dev/null +++ b/src/wallet/scriptpubkeyman.cpp @@ -0,0 +1,1428 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key_io.h> +#include <outputtype.h> +#include <script/descriptor.h> +#include <util/bip32.h> +#include <util/strencodings.h> +#include <util/translation.h> +#include <wallet/scriptpubkeyman.h> + +bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +{ + LOCK(cs_KeyStore); + error.clear(); + + // Generate a new key that is added to wallet + CPubKey new_key; + if (!GetKeyFromPool(new_key, type)) { + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + LearnRelatedScripts(new_key, type); + dest = GetDestinationForKey(new_key, type); + return true; +} + +typedef std::vector<unsigned char> valtype; + +namespace { + +/** + * This is an enum that tracks the execution context of a script, similar to + * SigVersion in script/interpreter. It is separate however because we want to + * distinguish between top-level scriptPubKey execution and P2SH redeemScript + * execution (a distinction that has no impact on consensus rules). + */ +enum class IsMineSigVersion +{ + TOP = 0, //!< scriptPubKey execution + P2SH = 1, //!< P2SH redeemScript + WITNESS_V0 = 2, //!< P2WSH witness script execution +}; + +/** + * This is an internal representation of isminetype + invalidity. + * Its order is significant, as we return the max of all explored + * possibilities. + */ +enum class IsMineResult +{ + NO = 0, //!< Not ours + WATCH_ONLY = 1, //!< Included in watch-only balance + SPENDABLE = 2, //!< Included in all balances + INVALID = 3, //!< Not spendable by anyone (uncompressed pubkey in segwit, P2SH inside P2SH or witness, witness inside witness) +}; + +bool PermitsUncompressed(IsMineSigVersion sigversion) +{ + return sigversion == IsMineSigVersion::TOP || sigversion == IsMineSigVersion::P2SH; +} + +bool HaveKeys(const std::vector<valtype>& pubkeys, const LegacyScriptPubKeyMan& keystore) +{ + for (const valtype& pubkey : pubkeys) { + CKeyID keyID = CPubKey(pubkey).GetID(); + if (!keystore.HaveKey(keyID)) return false; + } + return true; +} + +IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& scriptPubKey, IsMineSigVersion sigversion) +{ + IsMineResult ret = IsMineResult::NO; + + std::vector<valtype> vSolutions; + txnouttype whichType = Solver(scriptPubKey, vSolutions); + + CKeyID keyID; + switch (whichType) + { + case TX_NONSTANDARD: + case TX_NULL_DATA: + case TX_WITNESS_UNKNOWN: + break; + case TX_PUBKEY: + keyID = CPubKey(vSolutions[0]).GetID(); + if (!PermitsUncompressed(sigversion) && vSolutions[0].size() != 33) { + return IsMineResult::INVALID; + } + if (keystore.HaveKey(keyID)) { + ret = std::max(ret, IsMineResult::SPENDABLE); + } + break; + case TX_WITNESS_V0_KEYHASH: + { + if (sigversion == IsMineSigVersion::WITNESS_V0) { + // P2WPKH inside P2WSH is invalid. + return IsMineResult::INVALID; + } + if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { + // We do not support bare witness outputs unless the P2SH version of it would be + // acceptable as well. This protects against matching before segwit activates. + // This also applies to the P2WSH case. + break; + } + ret = std::max(ret, IsMineInner(keystore, GetScriptForDestination(PKHash(uint160(vSolutions[0]))), IsMineSigVersion::WITNESS_V0)); + break; + } + case TX_PUBKEYHASH: + keyID = CKeyID(uint160(vSolutions[0])); + if (!PermitsUncompressed(sigversion)) { + CPubKey pubkey; + if (keystore.GetPubKey(keyID, pubkey) && !pubkey.IsCompressed()) { + return IsMineResult::INVALID; + } + } + if (keystore.HaveKey(keyID)) { + ret = std::max(ret, IsMineResult::SPENDABLE); + } + break; + case TX_SCRIPTHASH: + { + if (sigversion != IsMineSigVersion::TOP) { + // P2SH inside P2WSH or P2SH is invalid. + return IsMineResult::INVALID; + } + CScriptID scriptID = CScriptID(uint160(vSolutions[0])); + CScript subscript; + if (keystore.GetCScript(scriptID, subscript)) { + ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::P2SH)); + } + break; + } + case TX_WITNESS_V0_SCRIPTHASH: + { + if (sigversion == IsMineSigVersion::WITNESS_V0) { + // P2WSH inside P2WSH is invalid. + return IsMineResult::INVALID; + } + if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { + break; + } + uint160 hash; + CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(hash.begin()); + CScriptID scriptID = CScriptID(hash); + CScript subscript; + if (keystore.GetCScript(scriptID, subscript)) { + ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::WITNESS_V0)); + } + break; + } + + case TX_MULTISIG: + { + // Never treat bare multisig outputs as ours (they can still be made watchonly-though) + if (sigversion == IsMineSigVersion::TOP) { + break; + } + + // Only consider transactions "mine" if we own ALL the + // keys involved. Multi-signature transactions that are + // partially owned (somebody else has a key that can spend + // them) enable spend-out-from-under-you attacks, especially + // in shared-wallet situations. + std::vector<valtype> keys(vSolutions.begin()+1, vSolutions.begin()+vSolutions.size()-1); + if (!PermitsUncompressed(sigversion)) { + for (size_t i = 0; i < keys.size(); i++) { + if (keys[i].size() != 33) { + return IsMineResult::INVALID; + } + } + } + if (HaveKeys(keys, keystore)) { + ret = std::max(ret, IsMineResult::SPENDABLE); + } + break; + } + } + + if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { + ret = std::max(ret, IsMineResult::WATCH_ONLY); + } + return ret; +} + +} // namespace + +isminetype LegacyScriptPubKeyMan::IsMine(const CScript& script) const +{ + switch (IsMineInner(*this, script, IsMineSigVersion::TOP)) { + case IsMineResult::INVALID: + case IsMineResult::NO: + return ISMINE_NO; + case IsMineResult::WATCH_ONLY: + return ISMINE_WATCH_ONLY; + case IsMineResult::SPENDABLE: + return ISMINE_SPENDABLE; + } + assert(false); +} + +bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys) +{ + { + LOCK(cs_KeyStore); + assert(mapKeys.empty()); + + bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys + bool keyFail = false; + CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); + for (; mi != mapCryptedKeys.end(); ++mi) + { + const CPubKey &vchPubKey = (*mi).second.first; + const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; + CKey key; + if (!DecryptKey(master_key, vchCryptedSecret, vchPubKey, key)) + { + keyFail = true; + break; + } + keyPass = true; + if (fDecryptionThoroughlyChecked) + break; + } + if (keyPass && keyFail) + { + LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); + throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); + } + if (keyFail || (!keyPass && !accept_no_keys)) + return false; + fDecryptionThoroughlyChecked = true; + } + return true; +} + +bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) +{ + LOCK(cs_KeyStore); + encrypted_batch = batch; + if (!mapCryptedKeys.empty()) { + encrypted_batch = nullptr; + return false; + } + + KeyMap keys_to_encrypt; + keys_to_encrypt.swap(mapKeys); // Clear mapKeys so AddCryptedKeyInner will succeed. + for (const KeyMap::value_type& mKey : keys_to_encrypt) + { + const CKey &key = mKey.second; + CPubKey vchPubKey = key.GetPubKey(); + CKeyingMaterial vchSecret(key.begin(), key.end()); + std::vector<unsigned char> vchCryptedSecret; + if (!EncryptSecret(master_key, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) { + encrypted_batch = nullptr; + return false; + } + if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) { + encrypted_batch = nullptr; + return false; + } + } + encrypted_batch = nullptr; + return true; +} + +bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) +{ + LOCK(cs_KeyStore); + if (!CanGetAddresses(internal)) { + return false; + } + + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { + return false; + } + address = GetDestinationForKey(keypool.vchPubKey, type); + return true; +} + +void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +{ + LOCK(cs_KeyStore); + // extract addresses and check if they match with an unused keypool key + for (const auto& keyid : GetAffectedKeys(script, *this)) { + std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); + if (mi != m_pool_key_to_index.end()) { + WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); + MarkReserveKeysAsUsed(mi->second); + + if (!TopUp()) { + WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); + } + } + } +} + +void LegacyScriptPubKeyMan::UpgradeKeyMetadata() +{ + LOCK(cs_KeyStore); + if (m_storage.IsLocked() || m_storage.IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { + return; + } + + std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(m_storage.GetDatabase()); + for (auto& meta_pair : mapKeyMetadata) { + CKeyMetadata& meta = meta_pair.second; + if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin + CKey key; + GetKey(meta.hd_seed_id, key); + CExtKey masterKey; + masterKey.SetSeed(key.begin(), key.size()); + // Add to map + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); + if (!ParseHDKeypath(meta.hdKeypath, meta.key_origin.path)) { + throw std::runtime_error("Invalid stored hdKeypath"); + } + meta.has_key_origin = true; + if (meta.nVersion < CKeyMetadata::VERSION_WITH_KEY_ORIGIN) { + meta.nVersion = CKeyMetadata::VERSION_WITH_KEY_ORIGIN; + } + + // Write meta to wallet + CPubKey pubkey; + if (GetPubKey(meta_pair.first, pubkey)) { + batch->WriteKeyMetadata(meta, pubkey, true); + } + } + } +} + +bool LegacyScriptPubKeyMan::SetupGeneration(bool force) +{ + if ((CanGenerateKeys() && !force) || m_storage.IsLocked()) { + return false; + } + + SetHDSeed(GenerateNewSeed()); + if (!NewKeyPool()) { + return false; + } + return true; +} + +bool LegacyScriptPubKeyMan::IsHDEnabled() const +{ + return !hdChain.seed_id.IsNull(); +} + +bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) +{ + LOCK(cs_KeyStore); + // Check if the keypool has keys + bool keypool_has_keys; + if (internal && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + keypool_has_keys = setInternalKeyPool.size() > 0; + } else { + keypool_has_keys = KeypoolCountExternalKeys() > 0; + } + // If the keypool doesn't have keys, check if we can generate them + if (!keypool_has_keys) { + return CanGenerateKeys(); + } + return keypool_has_keys; +} + +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) +{ + LOCK(cs_KeyStore); + error = ""; + bool hd_upgrade = false; + bool split_upgrade = false; + if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { + WalletLogPrintf("Upgrading wallet to HD\n"); + m_storage.SetMinVersion(FEATURE_HD); + + // generate a new master key + CPubKey masterPubKey = GenerateNewSeed(); + SetHDSeed(masterPubKey); + hd_upgrade = true; + } + // Upgrade to HD chain split if necessary + if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) && CHDChain::VERSION_HD_CHAIN_SPLIT) { + WalletLogPrintf("Upgrading wallet to use HD chain split\n"); + m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); + split_upgrade = FEATURE_HD_SPLIT > prev_version; + } + // Mark all keys currently in the keypool as pre-split + if (split_upgrade) { + MarkPreSplitKeys(); + } + // Regenerate the keypool if upgraded to HD + if (hd_upgrade) { + if (!TopUp()) { + error = _("Unable to generate keys").translated; + return false; + } + } + return true; +} + +bool LegacyScriptPubKeyMan::HavePrivateKeys() const +{ + LOCK(cs_KeyStore); + return !mapKeys.empty() || !mapCryptedKeys.empty(); +} + +void LegacyScriptPubKeyMan::RewriteDB() +{ + LOCK(cs_KeyStore); + setInternalKeyPool.clear(); + setExternalKeyPool.clear(); + m_pool_key_to_index.clear(); + // Note: can't top-up keypool here, because wallet is locked. + // User will be prompted to unlock wallet the next operation + // that requires a new key. +} + +static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, WalletBatch& batch) { + if (setKeyPool.empty()) { + return GetTime(); + } + + CKeyPool keypool; + int64_t nIndex = *(setKeyPool.begin()); + if (!batch.ReadPool(nIndex, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); + } + assert(keypool.vchPubKey.IsValid()); + return keypool.nTime; +} + +int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() +{ + LOCK(cs_KeyStore); + + WalletBatch batch(m_storage.GetDatabase()); + + // load oldest key from keypool, get time and return + int64_t oldestKey = GetOldestKeyTimeInPool(setExternalKeyPool, batch); + if (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + oldestKey = std::max(GetOldestKeyTimeInPool(setInternalKeyPool, batch), oldestKey); + if (!set_pre_split_keypool.empty()) { + oldestKey = std::max(GetOldestKeyTimeInPool(set_pre_split_keypool, batch), oldestKey); + } + } + + return oldestKey; +} + +size_t LegacyScriptPubKeyMan::KeypoolCountExternalKeys() +{ + LOCK(cs_KeyStore); + return setExternalKeyPool.size() + set_pre_split_keypool.size(); +} + +unsigned int LegacyScriptPubKeyMan::GetKeyPoolSize() const +{ + LOCK(cs_KeyStore); + return setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(); +} + +int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const +{ + LOCK(cs_KeyStore); + return nTimeFirstKey; +} + +std::unique_ptr<SigningProvider> LegacyScriptPubKeyMan::GetSigningProvider(const CScript& script) const +{ + return MakeUnique<LegacySigningProvider>(*this); +} + +bool LegacyScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sigdata) +{ + if (IsMine(script) != ISMINE_NO) { + // If it IsMine, we can always provide in some way + return true; + } else if (HaveCScript(CScriptID(script))) { + // We can still provide some stuff if we have the script, but IsMine failed because we don't have keys + return true; + } else { + // If, given the stuff in sigdata, we could make a valid sigature, then we can provide for this script + ProduceSignature(*this, DUMMY_SIGNATURE_CREATOR, script, sigdata); + if (!sigdata.signatures.empty()) { + // If we could make signatures, make sure we have a private key to actually make a signature + bool has_privkeys = false; + for (const auto& key_sig_pair : sigdata.signatures) { + has_privkeys |= HaveKey(key_sig_pair.first); + } + return has_privkeys; + } + return false; + } +} + +const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +{ + LOCK(cs_KeyStore); + + CKeyID key_id = GetKeyForDestination(*this, dest); + if (!key_id.IsNull()) { + auto it = mapKeyMetadata.find(key_id); + if (it != mapKeyMetadata.end()) { + return &it->second; + } + } + + CScript scriptPubKey = GetScriptForDestination(dest); + auto it = m_script_metadata.find(CScriptID(scriptPubKey)); + if (it != m_script_metadata.end()) { + return &it->second; + } + + return nullptr; +} + +uint256 LegacyScriptPubKeyMan::GetID() const +{ + return UINT256_ONE(); +} + +/** + * Update wallet first key creation time. This should be called whenever keys + * are added to the wallet, with the oldest key creation time. + */ +void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) +{ + AssertLockHeld(cs_KeyStore); + if (nCreateTime <= 1) { + // Cannot determine birthday information, so set the wallet birthday to + // the beginning of time. + nTimeFirstKey = 1; + } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { + nTimeFirstKey = nCreateTime; + } +} + +bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) +{ + return AddKeyPubKeyInner(key, pubkey); +} + +bool LegacyScriptPubKeyMan::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) +{ + LOCK(cs_KeyStore); + WalletBatch batch(m_storage.GetDatabase()); + return LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(batch, secret, pubkey); +} + +bool LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) +{ + AssertLockHeld(cs_KeyStore); + + // Make sure we aren't adding private keys to private key disabled wallets + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + + // FillableSigningProvider has no concept of wallet databases, but calls AddCryptedKey + // which is overridden below. To avoid flushes, the database handle is + // tunneled through to it. + bool needsDB = !encrypted_batch; + if (needsDB) { + encrypted_batch = &batch; + } + if (!AddKeyPubKeyInner(secret, pubkey)) { + if (needsDB) encrypted_batch = nullptr; + return false; + } + if (needsDB) encrypted_batch = nullptr; + + // check if we need to remove from watch-only + CScript script; + script = GetScriptForDestination(PKHash(pubkey)); + if (HaveWatchOnly(script)) { + RemoveWatchOnly(script); + } + script = GetScriptForRawPubKey(pubkey); + if (HaveWatchOnly(script)) { + RemoveWatchOnly(script); + } + + if (!m_storage.HasEncryptionKeys()) { + return batch.WriteKey(pubkey, + secret.GetPrivKey(), + mapKeyMetadata[pubkey.GetID()]); + } + m_storage.UnsetBlankWalletFlag(batch); + return true; +} + +bool LegacyScriptPubKeyMan::LoadCScript(const CScript& redeemScript) +{ + /* A sanity check was added in pull #3843 to avoid adding redeemScripts + * that never can be redeemed. However, old wallets may still contain + * these. Do not add them to the wallet and warn. */ + if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) + { + std::string strAddr = EncodeDestination(ScriptHash(redeemScript)); + WalletLogPrintf("%s: Warning: This wallet contains a redeemScript of size %i which exceeds maximum size %i thus can never be redeemed. Do not use address %s.\n", __func__, redeemScript.size(), MAX_SCRIPT_ELEMENT_SIZE, strAddr); + return true; + } + + return FillableSigningProvider::AddCScript(redeemScript); +} + +void LegacyScriptPubKeyMan::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) +{ + LOCK(cs_KeyStore); + UpdateTimeFirstKey(meta.nCreateTime); + mapKeyMetadata[keyID] = meta; +} + +void LegacyScriptPubKeyMan::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) +{ + LOCK(cs_KeyStore); + UpdateTimeFirstKey(meta.nCreateTime); + m_script_metadata[script_id] = meta; +} + +bool LegacyScriptPubKeyMan::AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey) +{ + LOCK(cs_KeyStore); + if (!m_storage.HasEncryptionKeys()) { + return FillableSigningProvider::AddKeyPubKey(key, pubkey); + } + + if (m_storage.IsLocked()) { + return false; + } + + std::vector<unsigned char> vchCryptedSecret; + CKeyingMaterial vchSecret(key.begin(), key.end()); + if (!EncryptSecret(m_storage.GetEncryptionKey(), vchSecret, pubkey.GetHash(), vchCryptedSecret)) { + return false; + } + + if (!AddCryptedKey(pubkey, vchCryptedSecret)) { + return false; + } + return true; +} + +bool LegacyScriptPubKeyMan::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +{ + return AddCryptedKeyInner(vchPubKey, vchCryptedSecret); +} + +bool LegacyScriptPubKeyMan::AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +{ + LOCK(cs_KeyStore); + assert(mapKeys.empty()); + + mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); + ImplicitlyLearnRelatedKeyScripts(vchPubKey); + return true; +} + +bool LegacyScriptPubKeyMan::AddCryptedKey(const CPubKey &vchPubKey, + const std::vector<unsigned char> &vchCryptedSecret) +{ + if (!AddCryptedKeyInner(vchPubKey, vchCryptedSecret)) + return false; + { + LOCK(cs_KeyStore); + if (encrypted_batch) + return encrypted_batch->WriteCryptedKey(vchPubKey, + vchCryptedSecret, + mapKeyMetadata[vchPubKey.GetID()]); + else + return WalletBatch(m_storage.GetDatabase()).WriteCryptedKey(vchPubKey, + vchCryptedSecret, + mapKeyMetadata[vchPubKey.GetID()]); + } +} + +bool LegacyScriptPubKeyMan::HaveWatchOnly(const CScript &dest) const +{ + LOCK(cs_KeyStore); + return setWatchOnly.count(dest) > 0; +} + +bool LegacyScriptPubKeyMan::HaveWatchOnly() const +{ + LOCK(cs_KeyStore); + return (!setWatchOnly.empty()); +} + +static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) +{ + std::vector<std::vector<unsigned char>> solutions; + return Solver(dest, solutions) == TX_PUBKEY && + (pubKeyOut = CPubKey(solutions[0])).IsFullyValid(); +} + +bool LegacyScriptPubKeyMan::RemoveWatchOnly(const CScript &dest) +{ + { + LOCK(cs_KeyStore); + setWatchOnly.erase(dest); + CPubKey pubKey; + if (ExtractPubKey(dest, pubKey)) { + mapWatchKeys.erase(pubKey.GetID()); + } + // Related CScripts are not removed; having superfluous scripts around is + // harmless (see comment in ImplicitlyLearnRelatedKeyScripts). + } + + if (!HaveWatchOnly()) + NotifyWatchonlyChanged(false); + if (!WalletBatch(m_storage.GetDatabase()).EraseWatchOnly(dest)) + return false; + + return true; +} + +bool LegacyScriptPubKeyMan::LoadWatchOnly(const CScript &dest) +{ + return AddWatchOnlyInMem(dest); +} + +bool LegacyScriptPubKeyMan::AddWatchOnlyInMem(const CScript &dest) +{ + LOCK(cs_KeyStore); + setWatchOnly.insert(dest); + CPubKey pubKey; + if (ExtractPubKey(dest, pubKey)) { + mapWatchKeys[pubKey.GetID()] = pubKey; + ImplicitlyLearnRelatedKeyScripts(pubKey); + } + return true; +} + +bool LegacyScriptPubKeyMan::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) +{ + if (!AddWatchOnlyInMem(dest)) + return false; + const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; + UpdateTimeFirstKey(meta.nCreateTime); + NotifyWatchonlyChanged(true); + if (batch.WriteWatchOnly(dest, meta)) { + m_storage.UnsetBlankWalletFlag(batch); + return true; + } + return false; +} + +bool LegacyScriptPubKeyMan::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) +{ + m_script_metadata[CScriptID(dest)].nCreateTime = create_time; + return AddWatchOnlyWithDB(batch, dest); +} + +bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest) +{ + WalletBatch batch(m_storage.GetDatabase()); + return AddWatchOnlyWithDB(batch, dest); +} + +bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTime) +{ + m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; + return AddWatchOnly(dest); +} + +void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) +{ + LOCK(cs_KeyStore); + if (!memonly && !WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + + hdChain = chain; +} + +bool LegacyScriptPubKeyMan::HaveKey(const CKeyID &address) const +{ + LOCK(cs_KeyStore); + if (!m_storage.HasEncryptionKeys()) { + return FillableSigningProvider::HaveKey(address); + } + return mapCryptedKeys.count(address) > 0; +} + +bool LegacyScriptPubKeyMan::GetKey(const CKeyID &address, CKey& keyOut) const +{ + LOCK(cs_KeyStore); + if (!m_storage.HasEncryptionKeys()) { + return FillableSigningProvider::GetKey(address, keyOut); + } + + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) + { + const CPubKey &vchPubKey = (*mi).second.first; + const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; + return DecryptKey(m_storage.GetEncryptionKey(), vchCryptedSecret, vchPubKey, keyOut); + } + return false; +} + +bool LegacyScriptPubKeyMan::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const +{ + CKeyMetadata meta; + { + LOCK(cs_KeyStore); + auto it = mapKeyMetadata.find(keyID); + if (it != mapKeyMetadata.end()) { + meta = it->second; + } + } + if (meta.has_key_origin) { + std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); + info.path = meta.key_origin.path; + } else { // Single pubkeys get the master fingerprint of themselves + std::copy(keyID.begin(), keyID.begin() + 4, info.fingerprint); + } + return true; +} + +bool LegacyScriptPubKeyMan::GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const +{ + LOCK(cs_KeyStore); + WatchKeyMap::const_iterator it = mapWatchKeys.find(address); + if (it != mapWatchKeys.end()) { + pubkey_out = it->second; + return true; + } + return false; +} + +bool LegacyScriptPubKeyMan::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const +{ + LOCK(cs_KeyStore); + if (!m_storage.HasEncryptionKeys()) { + if (!FillableSigningProvider::GetPubKey(address, vchPubKeyOut)) { + return GetWatchPubKey(address, vchPubKeyOut); + } + return true; + } + + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) + { + vchPubKeyOut = (*mi).second.first; + return true; + } + // Check for watch-only pubkeys + return GetWatchPubKey(address, vchPubKeyOut); +} + +CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) +{ + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); + AssertLockHeld(cs_KeyStore); + bool fCompressed = m_storage.CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets + + CKey secret; + + // Create new metadata + int64_t nCreationTime = GetTime(); + CKeyMetadata metadata(nCreationTime); + + // use HD key derivation if HD was enabled during wallet creation and a seed is present + if (IsHDEnabled()) { + DeriveNewChildKey(batch, metadata, secret, (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); + } else { + secret.MakeNewKey(fCompressed); + } + + // Compressed public keys were introduced in version 0.6.0 + if (fCompressed) { + m_storage.SetMinVersion(FEATURE_COMPRPUBKEY); + } + + CPubKey pubkey = secret.GetPubKey(); + assert(secret.VerifyPubKey(pubkey)); + + mapKeyMetadata[pubkey.GetID()] = metadata; + UpdateTimeFirstKey(nCreationTime); + + if (!AddKeyPubKeyWithDB(batch, secret, pubkey)) { + throw std::runtime_error(std::string(__func__) + ": AddKey failed"); + } + return pubkey; +} + +const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; + +void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal) +{ + // for now we use a fixed keypath scheme of m/0'/0'/k + CKey seed; //seed (256bit) + CExtKey masterKey; //hd master key + CExtKey accountKey; //key at m/0' + CExtKey chainChildKey; //key at m/0'/0' (external) or m/0'/1' (internal) + CExtKey childKey; //key at m/0'/0'/<n>' + + // try to get the seed + if (!GetKey(hdChain.seed_id, seed)) + throw std::runtime_error(std::string(__func__) + ": seed not found"); + + masterKey.SetSeed(seed.begin(), seed.size()); + + // derive m/0' + // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) + masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); + + // derive m/0'/0' (external chain) OR m/0'/1' (internal chain) + assert(internal ? m_storage.CanSupportFeature(FEATURE_HD_SPLIT) : true); + accountKey.Derive(chainChildKey, BIP32_HARDENED_KEY_LIMIT+(internal ? 1 : 0)); + + // derive child key at next index, skip keys already known to the wallet + do { + // always derive hardened keys + // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range + // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 + if (internal) { + chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hdChain.nInternalChainCounter++; + } + else { + chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hdChain.nExternalChainCounter++; + } + } while (HaveKey(childKey.key.GetPubKey().GetID())); + secret = childKey.key; + metadata.hd_seed_id = hdChain.seed_id; + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); + metadata.has_key_origin = true; + // update the chain model in the database + if (!batch.WriteHDChain(hdChain)) + throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); +} + +void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) +{ + LOCK(cs_KeyStore); + if (keypool.m_pre_split) { + set_pre_split_keypool.insert(nIndex); + } else if (keypool.fInternal) { + setInternalKeyPool.insert(nIndex); + } else { + setExternalKeyPool.insert(nIndex); + } + m_max_keypool_index = std::max(m_max_keypool_index, nIndex); + m_pool_key_to_index[keypool.vchPubKey.GetID()] = nIndex; + + // If no metadata exists yet, create a default with the pool key's + // creation time. Note that this may be overwritten by actually + // stored metadata for that key later, which is fine. + CKeyID keyid = keypool.vchPubKey.GetID(); + if (mapKeyMetadata.count(keyid) == 0) + mapKeyMetadata[keyid] = CKeyMetadata(keypool.nTime); +} + +bool LegacyScriptPubKeyMan::CanGenerateKeys() +{ + // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) + LOCK(cs_KeyStore); + return IsHDEnabled() || !m_storage.CanSupportFeature(FEATURE_HD); +} + +CPubKey LegacyScriptPubKeyMan::GenerateNewSeed() +{ + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + CKey key; + key.MakeNewKey(true); + return DeriveNewSeed(key); +} + +CPubKey LegacyScriptPubKeyMan::DeriveNewSeed(const CKey& key) +{ + int64_t nCreationTime = GetTime(); + CKeyMetadata metadata(nCreationTime); + + // calculate the seed + CPubKey seed = key.GetPubKey(); + assert(key.VerifyPubKey(seed)); + + // set the hd keypath to "s" -> Seed, refers the seed to itself + metadata.hdKeypath = "s"; + metadata.has_key_origin = false; + metadata.hd_seed_id = seed.GetID(); + + { + LOCK(cs_KeyStore); + + // mem store the metadata + mapKeyMetadata[seed.GetID()] = metadata; + + // write the key&metadata to the database + if (!AddKeyPubKey(key, seed)) + throw std::runtime_error(std::string(__func__) + ": AddKeyPubKey failed"); + } + + return seed; +} + +void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) +{ + LOCK(cs_KeyStore); + // store the keyid (hash160) together with + // the child index counter in the database + // as a hdchain object + CHDChain newHdChain; + newHdChain.nVersion = m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; + newHdChain.seed_id = seed.GetID(); + SetHDChain(newHdChain, false); + NotifyCanGetAddressesChanged(); + WalletBatch batch(m_storage.GetDatabase()); + m_storage.UnsetBlankWalletFlag(batch); +} + +/** + * Mark old keypool keys as used, + * and generate all new keys + */ +bool LegacyScriptPubKeyMan::NewKeyPool() +{ + if (m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + return false; + } + { + LOCK(cs_KeyStore); + WalletBatch batch(m_storage.GetDatabase()); + + for (const int64_t nIndex : setInternalKeyPool) { + batch.ErasePool(nIndex); + } + setInternalKeyPool.clear(); + + for (const int64_t nIndex : setExternalKeyPool) { + batch.ErasePool(nIndex); + } + setExternalKeyPool.clear(); + + for (const int64_t nIndex : set_pre_split_keypool) { + batch.ErasePool(nIndex); + } + set_pre_split_keypool.clear(); + + m_pool_key_to_index.clear(); + + if (!TopUp()) { + return false; + } + WalletLogPrintf("LegacyScriptPubKeyMan::NewKeyPool rewrote keypool\n"); + } + return true; +} + +bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) +{ + if (!CanGenerateKeys()) { + return false; + } + { + LOCK(cs_KeyStore); + + if (m_storage.IsLocked()) return false; + + // Top up key pool + unsigned int nTargetSize; + if (kpSize > 0) + nTargetSize = kpSize; + else + nTargetSize = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); + + // count amount of available keys (internal, external) + // make sure the keypool of external and internal keys fits the user selected target (-keypool) + int64_t missingExternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setExternalKeyPool.size(), (int64_t) 0); + int64_t missingInternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setInternalKeyPool.size(), (int64_t) 0); + + if (!IsHDEnabled() || !m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) + { + // don't create extra internal keys + missingInternal = 0; + } + bool internal = false; + WalletBatch batch(m_storage.GetDatabase()); + for (int64_t i = missingInternal + missingExternal; i--;) + { + if (i < missingInternal) { + internal = true; + } + + CPubKey pubkey(GenerateNewKey(batch, internal)); + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + } + if (missingInternal + missingExternal > 0) { + WalletLogPrintf("keypool added %d keys (%d internal), size=%u (%u internal)\n", missingInternal + missingExternal, missingInternal, setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(), setInternalKeyPool.size()); + } + } + NotifyCanGetAddressesChanged(); + return true; +} + +void LegacyScriptPubKeyMan::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) +{ + LOCK(cs_KeyStore); + assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? + int64_t index = ++m_max_keypool_index; + if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { + throw std::runtime_error(std::string(__func__) + ": writing imported pubkey failed"); + } + if (internal) { + setInternalKeyPool.insert(index); + } else { + setExternalKeyPool.insert(index); + } + m_pool_key_to_index[pubkey.GetID()] = index; +} + +void LegacyScriptPubKeyMan::KeepDestination(int64_t nIndex, const OutputType& type) +{ + // Remove from key pool + WalletBatch batch(m_storage.GetDatabase()); + batch.ErasePool(nIndex); + CPubKey pubkey; + bool have_pk = GetPubKey(m_index_to_reserved_key.at(nIndex), pubkey); + assert(have_pk); + LearnRelatedScripts(pubkey, type); + m_index_to_reserved_key.erase(nIndex); + WalletLogPrintf("keypool keep %d\n", nIndex); +} + +void LegacyScriptPubKeyMan::ReturnDestination(int64_t nIndex, bool fInternal, const CTxDestination&) +{ + // Return to key pool + { + LOCK(cs_KeyStore); + if (fInternal) { + setInternalKeyPool.insert(nIndex); + } else if (!set_pre_split_keypool.empty()) { + set_pre_split_keypool.insert(nIndex); + } else { + setExternalKeyPool.insert(nIndex); + } + CKeyID& pubkey_id = m_index_to_reserved_key.at(nIndex); + m_pool_key_to_index[pubkey_id] = nIndex; + m_index_to_reserved_key.erase(nIndex); + NotifyCanGetAddressesChanged(); + } + WalletLogPrintf("keypool return %d\n", nIndex); +} + +bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, const OutputType type, bool internal) +{ + if (!CanGetAddresses(internal)) { + return false; + } + + CKeyPool keypool; + { + LOCK(cs_KeyStore); + int64_t nIndex; + if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (m_storage.IsLocked()) return false; + WalletBatch batch(m_storage.GetDatabase()); + result = GenerateNewKey(batch, internal); + return true; + } + KeepDestination(nIndex, type); + result = keypool.vchPubKey; + } + return true; +} + +bool LegacyScriptPubKeyMan::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal) +{ + nIndex = -1; + keypool.vchPubKey = CPubKey(); + { + LOCK(cs_KeyStore); + + bool fReturningInternal = fRequestedInternal; + fReturningInternal &= (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) || m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + bool use_split_keypool = set_pre_split_keypool.empty(); + std::set<int64_t>& setKeyPool = use_split_keypool ? (fReturningInternal ? setInternalKeyPool : setExternalKeyPool) : set_pre_split_keypool; + + // Get the oldest key + if (setKeyPool.empty()) { + return false; + } + + WalletBatch batch(m_storage.GetDatabase()); + + auto it = setKeyPool.begin(); + nIndex = *it; + setKeyPool.erase(it); + if (!batch.ReadPool(nIndex, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read failed"); + } + CPubKey pk; + if (!GetPubKey(keypool.vchPubKey.GetID(), pk)) { + throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); + } + // If the key was pre-split keypool, we don't care about what type it is + if (use_split_keypool && keypool.fInternal != fReturningInternal) { + throw std::runtime_error(std::string(__func__) + ": keypool entry misclassified"); + } + if (!keypool.vchPubKey.IsValid()) { + throw std::runtime_error(std::string(__func__) + ": keypool entry invalid"); + } + + assert(m_index_to_reserved_key.count(nIndex) == 0); + m_index_to_reserved_key[nIndex] = keypool.vchPubKey.GetID(); + m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); + WalletLogPrintf("keypool reserve %d\n", nIndex); + } + NotifyCanGetAddressesChanged(); + return true; +} + +void LegacyScriptPubKeyMan::LearnRelatedScripts(const CPubKey& key, OutputType type) +{ + if (key.IsCompressed() && (type == OutputType::P2SH_SEGWIT || type == OutputType::BECH32)) { + CTxDestination witdest = WitnessV0KeyHash(key.GetID()); + CScript witprog = GetScriptForDestination(witdest); + // Make sure the resulting program is solvable. + assert(IsSolvable(*this, witprog)); + AddCScript(witprog); + } +} + +void LegacyScriptPubKeyMan::LearnAllRelatedScripts(const CPubKey& key) +{ + // OutputType::P2SH_SEGWIT always adds all necessary scripts for all types. + LearnRelatedScripts(key, OutputType::P2SH_SEGWIT); +} + +void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) +{ + AssertLockHeld(cs_KeyStore); + bool internal = setInternalKeyPool.count(keypool_id); + if (!internal) assert(setExternalKeyPool.count(keypool_id) || set_pre_split_keypool.count(keypool_id)); + std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); + auto it = setKeyPool->begin(); + + WalletBatch batch(m_storage.GetDatabase()); + while (it != std::end(*setKeyPool)) { + const int64_t& index = *(it); + if (index > keypool_id) break; // set*KeyPool is ordered + + CKeyPool keypool; + if (batch.ReadPool(index, keypool)) { //TODO: This should be unnecessary + m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); + } + LearnAllRelatedScripts(keypool.vchPubKey); + batch.ErasePool(index); + WalletLogPrintf("keypool index %d removed\n", index); + it = setKeyPool->erase(it); + } +} + +std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider) +{ + std::vector<CScript> dummy; + FlatSigningProvider out; + InferDescriptor(spk, provider)->Expand(0, DUMMY_SIGNING_PROVIDER, dummy, out); + std::vector<CKeyID> ret; + for (const auto& entry : out.pubkeys) { + ret.push_back(entry.first); + } + return ret; +} + +void LegacyScriptPubKeyMan::MarkPreSplitKeys() +{ + WalletBatch batch(m_storage.GetDatabase()); + for (auto it = setExternalKeyPool.begin(); it != setExternalKeyPool.end();) { + int64_t index = *it; + CKeyPool keypool; + if (!batch.ReadPool(index, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read keypool entry failed"); + } + keypool.m_pre_split = true; + if (!batch.WritePool(index, keypool)) { + throw std::runtime_error(std::string(__func__) + ": writing modified keypool entry failed"); + } + set_pre_split_keypool.insert(index); + it = setExternalKeyPool.erase(it); + } +} + +bool LegacyScriptPubKeyMan::AddCScript(const CScript& redeemScript) +{ + WalletBatch batch(m_storage.GetDatabase()); + return AddCScriptWithDB(batch, redeemScript); +} + +bool LegacyScriptPubKeyMan::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript) +{ + if (!FillableSigningProvider::AddCScript(redeemScript)) + return false; + if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) { + m_storage.UnsetBlankWalletFlag(batch); + return true; + } + return false; +} + +bool LegacyScriptPubKeyMan::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) +{ + LOCK(cs_KeyStore); + std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); + mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; + mapKeyMetadata[pubkey.GetID()].has_key_origin = true; + mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); + return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); +} + +bool LegacyScriptPubKeyMan::ImportScripts(const std::set<CScript> scripts, int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const auto& entry : scripts) { + CScriptID id(entry); + if (HaveCScript(id)) { + WalletLogPrintf("Already have script %s, skipping\n", HexStr(entry)); + continue; + } + if (!AddCScriptWithDB(batch, entry)) { + return false; + } + + if (timestamp > 0) { + m_script_metadata[CScriptID(entry)].nCreateTime = timestamp; + } + } + if (timestamp > 0) { + UpdateTimeFirstKey(timestamp); + } + + return true; +} + +bool LegacyScriptPubKeyMan::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const auto& entry : privkey_map) { + const CKey& key = entry.second; + CPubKey pubkey = key.GetPubKey(); + const CKeyID& id = entry.first; + assert(key.VerifyPubKey(pubkey)); + // Skip if we already have the key + if (HaveKey(id)) { + WalletLogPrintf("Already have key with pubkey %s, skipping\n", HexStr(pubkey)); + continue; + } + mapKeyMetadata[id].nCreateTime = timestamp; + // If the private key is not present in the wallet, insert it. + if (!AddKeyPubKeyWithDB(batch, key, pubkey)) { + return false; + } + UpdateTimeFirstKey(timestamp); + } + return true; +} + +bool LegacyScriptPubKeyMan::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const auto& entry : key_origins) { + AddKeyOriginWithDB(batch, entry.second.first, entry.second.second); + } + for (const CKeyID& id : ordered_pubkeys) { + auto entry = pubkey_map.find(id); + if (entry == pubkey_map.end()) { + continue; + } + const CPubKey& pubkey = entry->second; + CPubKey temp; + if (GetPubKey(id, temp)) { + // Already have pubkey, skipping + WalletLogPrintf("Already have pubkey %s, skipping\n", HexStr(temp)); + continue; + } + if (!AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) { + return false; + } + mapKeyMetadata[id].nCreateTime = timestamp; + + // Add to keypool only works with pubkeys + if (add_keypool) { + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + NotifyCanGetAddressesChanged(); + } + } + return true; +} + +bool LegacyScriptPubKeyMan::ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const CScript& script : script_pub_keys) { + if (!have_solving_data || !IsMine(script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated + if (!AddWatchOnlyWithDB(batch, script, timestamp)) { + return false; + } + } + } + return true; +} + +std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const +{ + LOCK(cs_KeyStore); + if (!m_storage.HasEncryptionKeys()) { + return FillableSigningProvider::GetKeys(); + } + std::set<CKeyID> set_address; + for (const auto& mi : mapCryptedKeys) { + set_address.insert(mi.first); + } + return set_address; +} diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h new file mode 100644 index 0000000000..7b1c023bc9 --- /dev/null +++ b/src/wallet/scriptpubkeyman.h @@ -0,0 +1,466 @@ +// Copyright (c) 2019-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_SCRIPTPUBKEYMAN_H +#define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H + +#include <script/signingprovider.h> +#include <script/standard.h> +#include <wallet/crypter.h> +#include <wallet/ismine.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> + +#include <boost/signals2/signal.hpp> + +enum class OutputType; + +// Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. +// It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as +// wallet flags, wallet version, encryption keys, encryption status, and the database itself. This allows a +// ScriptPubKeyMan to have callbacks into CWallet without causing a circular dependency. +// WalletStorage should be the same for all ScriptPubKeyMans of a wallet. +class WalletStorage +{ +public: + virtual ~WalletStorage() = default; + virtual const std::string GetDisplayName() const = 0; + virtual WalletDatabase& GetDatabase() = 0; + virtual bool IsWalletFlagSet(uint64_t) const = 0; + virtual void UnsetBlankWalletFlag(WalletBatch&) = 0; + virtual bool CanSupportFeature(enum WalletFeature) const = 0; + virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr, bool = false) = 0; + virtual const CKeyingMaterial& GetEncryptionKey() const = 0; + virtual bool HasEncryptionKeys() const = 0; + virtual bool IsLocked() const = 0; +}; + +//! Default for -keypool +static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; + +std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider); + +/** A key from a CWallet's keypool + * + * The wallet holds one (for pre HD-split wallets) or several keypools. These + * are sets of keys that have not yet been used to provide addresses or receive + * change. + * + * The Bitcoin Core wallet was originally a collection of unrelated private + * keys with their associated addresses. If a non-HD wallet generated a + * key/address, gave that address out and then restored a backup from before + * that key's generation, then any funds sent to that address would be + * lost definitively. + * + * The keypool was implemented to avoid this scenario (commit: 10384941). The + * wallet would generate a set of keys (100 by default). When a new public key + * was required, either to give out as an address or to use in a change output, + * it would be drawn from the keypool. The keypool would then be topped up to + * maintain 100 keys. This ensured that as long as the wallet hadn't used more + * than 100 keys since the previous backup, all funds would be safe, since a + * restored wallet would be able to scan for all owned addresses. + * + * A keypool also allowed encrypted wallets to give out addresses without + * having to be decrypted to generate a new private key. + * + * With the introduction of HD wallets (commit: f1902510), the keypool + * essentially became an address look-ahead pool. Restoring old backups can no + * longer definitively lose funds as long as the addresses used were from the + * wallet's HD seed (since all private keys can be rederived from the seed). + * However, if many addresses were used since the backup, then the wallet may + * not know how far ahead in the HD chain to look for its addresses. The + * keypool is used to implement a 'gap limit'. The keypool maintains a set of + * keys (by default 1000) ahead of the last used key and scans for the + * addresses of those keys. This avoids the risk of not seeing transactions + * involving the wallet's addresses, or of re-using the same address. + * In the unlikely case where none of the addresses in the `gap limit` are + * used on-chain, the look-ahead will not be incremented to keep + * a constant size and addresses beyond this range will not be detected by an + * old backup. For this reason, it is not recommended to decrease keypool size + * lower than default value. + * + * The HD-split wallet feature added a second keypool (commit: 02592f4c). There + * is an external keypool (for addresses to hand out) and an internal keypool + * (for change addresses). + * + * Keypool keys are stored in the wallet/keystore's keymap. The keypool data is + * stored as sets of indexes in the wallet (setInternalKeyPool, + * setExternalKeyPool and set_pre_split_keypool), and a map from the key to the + * index (m_pool_key_to_index). The CKeyPool object is used to + * serialize/deserialize the pool data to/from the database. + */ +class CKeyPool +{ +public: + //! The time at which the key was generated. Set in AddKeypoolPubKeyWithDB + int64_t nTime; + //! The public key + CPubKey vchPubKey; + //! Whether this keypool entry is in the internal keypool (for change outputs) + bool fInternal; + //! Whether this key was generated for a keypool before the wallet was upgraded to HD-split + bool m_pre_split; + + CKeyPool(); + CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) + READWRITE(nVersion); + READWRITE(nTime); + READWRITE(vchPubKey); + if (ser_action.ForRead()) { + try { + READWRITE(fInternal); + } + catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; + } + try { + READWRITE(m_pre_split); + } + catch (std::ios_base::failure&) { + /* flag as postsplit address if we can't read the m_pre_split boolean + (this will be the case for any wallet that upgrades to HD chain split)*/ + m_pre_split = false; + } + } + else { + READWRITE(fInternal); + READWRITE(m_pre_split); + } + } +}; + +/* + * A class implementing ScriptPubKeyMan manages some (or all) scriptPubKeys used in a wallet. + * It contains the scripts and keys related to the scriptPubKeys it manages. + * A ScriptPubKeyMan will be able to give out scriptPubKeys to be used, as well as marking + * when a scriptPubKey has been used. It also handles when and how to store a scriptPubKey + * and its related scripts and keys, including encryption. + */ +class ScriptPubKeyMan +{ +protected: + WalletStorage& m_storage; + +public: + ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} + virtual ~ScriptPubKeyMan() {}; + virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { return false; } + virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } + + //! Check that the given decryption key is valid for this ScriptPubKeyMan, i.e. it decrypts all of the keys handled by it. + virtual bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) { return false; } + virtual bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) { return false; } + + virtual bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) { return false; } + virtual void KeepDestination(int64_t index, const OutputType& type) {} + virtual void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) {} + + /** Fills internal address pool. Use within ScriptPubKeyMan implementations should be used sparingly and only + * when something from the address pool is removed, excluding GetNewDestination and GetReservedDestination. + * External wallet code is primarily responsible for topping up prior to fetching new addresses + */ + virtual bool TopUp(unsigned int size = 0) { return false; } + + //! Mark unused addresses as being used + virtual void MarkUnusedAddresses(const CScript& script) {} + + /** Sets up the key generation stuff, i.e. generates new HD seeds and sets them as active. + * Returns false if already setup or setup fails, true if setup is successful + * Set force=true to make it re-setup if already setup, used for upgrades + */ + virtual bool SetupGeneration(bool force = false) { return false; } + + /* Returns true if HD is enabled */ + virtual bool IsHDEnabled() const { return false; } + + /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ + virtual bool CanGetAddresses(bool internal = false) { return false; } + + /** Upgrades the wallet to the specified version */ + virtual bool Upgrade(int prev_version, std::string& error) { return false; } + + virtual bool HavePrivateKeys() const { return false; } + + //! The action to do when the DB needs rewrite + virtual void RewriteDB() {} + + virtual int64_t GetOldestKeyPoolTime() { return GetTime(); } + + virtual size_t KeypoolCountExternalKeys() { return 0; } + virtual unsigned int GetKeyPoolSize() const { return 0; } + + virtual int64_t GetTimeFirstKey() const { return 0; } + + virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } + + virtual std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const { return nullptr; } + + /** Whether this ScriptPubKeyMan can provide a SigningProvider (via GetSigningProvider) that, combined with + * sigdata, can produce a valid signature. + */ + virtual bool CanProvide(const CScript& script, SignatureData& sigdata) { return false; } + + virtual uint256 GetID() const { return uint256(); } + + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ + template<typename... Params> + void WalletLogPrintf(std::string fmt, Params... parameters) const { + LogPrintf(("%s " + fmt).c_str(), m_storage.GetDisplayName(), parameters...); + }; + + /** Watch-only address added */ + boost::signals2::signal<void (bool fHaveWatchOnly)> NotifyWatchonlyChanged; + + /** Keypool has new keys */ + boost::signals2::signal<void ()> NotifyCanGetAddressesChanged; +}; + +class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProvider +{ +private: + //! keeps track of whether Unlock has run a thorough check before + bool fDecryptionThoroughlyChecked = false; + + using WatchOnlySet = std::set<CScript>; + using WatchKeyMap = std::map<CKeyID, CPubKey>; + + WalletBatch *encrypted_batch GUARDED_BY(cs_KeyStore) = nullptr; + + using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; + + CryptedKeyMap mapCryptedKeys GUARDED_BY(cs_KeyStore); + WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); + WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); + + int64_t nTimeFirstKey GUARDED_BY(cs_KeyStore) = 0; + + bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); + bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + + /** + * Private version of AddWatchOnly method which does not accept a + * timestamp, and which will reset the wallet's nTimeFirstKey value to 1 if + * the watch key did not previously have a timestamp associated with it. + * Because this is an inherited virtual method, it is accessible despite + * being marked private, but it is marked private anyway to encourage use + * of the other AddWatchOnly which accepts a timestamp and sets + * nTimeFirstKey more intelligently for more efficient rescans. + */ + bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool AddWatchOnlyInMem(const CScript &dest); + //! Adds a watch-only address to the store, and saves it to disk. + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + //! Adds a key to the store, and saves it to disk. + bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); + + //! Adds a script to the store and saves it to disk + bool AddCScriptWithDB(WalletBatch& batch, const CScript& script); + + /** Add a KeyOriginInfo to the wallet */ + bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); + + /* the HD chain data model (external chain counters) */ + CHDChain hdChain; + + /* HD derive new child key (on internal or external chain) */ + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_KeyStore); + std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_KeyStore); + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_KeyStore); + int64_t m_max_keypool_index GUARDED_BY(cs_KeyStore) = 0; + std::map<CKeyID, int64_t> m_pool_key_to_index; + // Tracks keypool indexes to CKeyIDs of keys that have been taken out of the keypool but may be returned to it + std::map<int64_t, CKeyID> m_index_to_reserved_key; + + //! Fetches a key from the keypool + bool GetKeyFromPool(CPubKey &key, const OutputType type, bool internal = false); + + /** + * Reserves a key from the keypool and sets nIndex to its index + * + * @param[out] nIndex the index of the key in keypool + * @param[out] keypool the keypool the key was drawn from, which could be the + * the pre-split pool if present, or the internal or external pool + * @param fRequestedInternal true if the caller would like the key drawn + * from the internal keypool, false if external is preferred + * + * @return true if succeeded, false if failed due to empty keypool + * @throws std::runtime_error if keypool read failed, key was invalid, + * was not found in the wallet, or was misclassified in the internal + * or external keypool + */ + bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); + +public: + using ScriptPubKeyMan::ScriptPubKeyMan; + + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + isminetype IsMine(const CScript& script) const override; + + bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; + bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) override; + + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) override; + void KeepDestination(int64_t index, const OutputType& type) override; + void ReturnDestination(int64_t index, bool internal, const CTxDestination&) override; + + bool TopUp(unsigned int size = 0) override; + + void MarkUnusedAddresses(const CScript& script) override; + + //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo + void UpgradeKeyMetadata(); + + bool IsHDEnabled() const override; + + bool SetupGeneration(bool force = false) override; + + bool Upgrade(int prev_version, std::string& error) override; + + bool HavePrivateKeys() const override; + + void RewriteDB() override; + + int64_t GetOldestKeyPoolTime() override; + size_t KeypoolCountExternalKeys() override; + unsigned int GetKeyPoolSize() const override; + + int64_t GetTimeFirstKey() const override; + + const CKeyMetadata* GetMetadata(const CTxDestination& dest) const override; + + bool CanGetAddresses(bool internal = false) override; + + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const override; + + bool CanProvide(const CScript& script, SignatureData& sigdata) override; + + uint256 GetID() const override; + + // Map from Key ID to key metadata. + std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); + + // Map from Script ID to key metadata (for watch-only keys). + std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_KeyStore); + + //! Adds a key to the store, and saves it to disk. + bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; + //! Adds a key to the store, without saving it to disk (used by LoadWallet) + bool LoadKey(const CKey& key, const CPubKey &pubkey); + //! Adds an encrypted key to the store, and saves it to disk. + bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) + bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + //! Adds a CScript to the store + bool LoadCScript(const CScript& redeemScript); + //! Load metadata (used by LoadWallet) + void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); + void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); + //! Generate a new key + CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + /* Set the HD chain model (chain child index counters) */ + void SetHDChain(const CHDChain& chain, bool memonly); + const CHDChain& GetHDChain() const { return hdChain; } + + //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) + bool LoadWatchOnly(const CScript &dest); + //! Returns whether the watch-only script is in the wallet + bool HaveWatchOnly(const CScript &dest) const; + //! Returns whether there are any watch-only things in the wallet + bool HaveWatchOnly() const; + //! Remove a watch only script from the keystore + bool RemoveWatchOnly(const CScript &dest); + bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + //! Fetches a pubkey from mapWatchKeys if it exists there + bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; + + /* SigningProvider overrides */ + bool HaveKey(const CKeyID &address) const override; + bool GetKey(const CKeyID &address, CKey& keyOut) const override; + bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; + bool AddCScript(const CScript& redeemScript) override; + bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + + //! Load a keypool entry + void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool); + bool NewKeyPool(); + void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + + /* Returns true if the wallet can generate new keys */ + bool CanGenerateKeys(); + + /* Generates a new HD seed (will not be activated) */ + CPubKey GenerateNewSeed(); + + /* Derives a new HD seed (will not be activated) */ + CPubKey DeriveNewSeed(const CKey& key); + + /* Set the current HD seed (will reset the chain child index counters) + Sets the seed's version based on the current wallet version (so the + caller must ensure the current wallet version is correct before calling + this function). */ + void SetHDSeed(const CPubKey& key); + + /** + * Explicitly make the wallet learn the related scripts for outputs to the + * given key. This is purely to make the wallet file compatible with older + * software, as FillableSigningProvider automatically does this implicitly for all + * keys now. + */ + void LearnRelatedScripts(const CPubKey& key, OutputType); + + /** + * Same as LearnRelatedScripts, but when the OutputType is not known (and could + * be anything). + */ + void LearnAllRelatedScripts(const CPubKey& key); + + /** + * Marks all keys in the keypool up to and including reserve_key as used. + */ + void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } + + std::set<CKeyID> GetKeys() const override; +}; + +/** Wraps a LegacyScriptPubKeyMan so that it can be returned in a new unique_ptr */ +class LegacySigningProvider : public SigningProvider +{ +private: + const LegacyScriptPubKeyMan& m_spk_man; +public: + LegacySigningProvider(const LegacyScriptPubKeyMan& spk_man) : m_spk_man(spk_man) {} + + bool GetCScript(const CScriptID &scriptid, CScript& script) const override { return m_spk_man.GetCScript(scriptid, script); } + bool HaveCScript(const CScriptID &scriptid) const override { return m_spk_man.HaveCScript(scriptid); } + bool GetPubKey(const CKeyID &address, CPubKey& pubkey) const override { return m_spk_man.GetPubKey(address, pubkey); } + bool GetKey(const CKeyID &address, CKey& key) const override { return m_spk_man.GetKey(address, key); } + bool HaveKey(const CKeyID &address) const override { return m_spk_man.HaveKey(address); } + bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override { return m_spk_man.GetKeyOrigin(keyid, info); } +}; + +#endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 9e7f0ed773..d65a0e9075 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -2,13 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <node/context.h> #include <wallet/wallet.h> #include <wallet/coinselection.h> #include <wallet/coincontrol.h> #include <amount.h> #include <primitives/transaction.h> #include <random.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> @@ -28,7 +29,8 @@ std::vector<std::unique_ptr<CWalletTx>> wtxn; typedef std::set<CInputCoin> CoinSet; static std::vector<COutput> vCoins; -static auto testChain = interfaces::MakeChain(); +static NodeContext testNode; +static auto testChain = interfaces::MakeChain(testNode); static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); static CAmount balance = 0; @@ -53,7 +55,7 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set) set.emplace(MakeTransactionRef(tx), nInput); } -static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0) +static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) { balance += nValue; static int nextLockTime = 0; @@ -61,21 +63,32 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa tx.nLockTime = nextLockTime++; // so all transactions get different hashes tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; + if (spendable) { + CTxDestination dest; + std::string error; + assert(wallet.GetNewDestination(OutputType::BECH32, "", dest, error)); + tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); + } if (fIsFromMe) { // IsFromMe() returns (GetDebit() > 0), and GetDebit() is 0 if vin.empty(), // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); + std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&wallet, MakeTransactionRef(std::move(tx))); if (fIsFromMe) { wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); + wtx->m_is_cache_empty = false; } COutput output(wtx.get(), nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); - testWallet.AddToWallet(*wtx.get()); + wallet.AddToWallet(*wtx.get()); wtxn.emplace_back(std::move(wtx)); } +static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) +{ + add_coin(testWallet, nValue, nAge, fIsFromMe, nInput, spendable); +} static void empty_wallet(void) { @@ -123,6 +136,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) { LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); // Setup std::vector<CInputCoin> utxo_pool; @@ -250,17 +264,34 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); - // Make sure that we aren't using BnB when there are preset inputs + // Test fees subtracted from output: empty_wallet(); - add_coin(5 * CENT); - add_coin(3 * CENT); - add_coin(2 * CENT); - CCoinControl coin_control; - coin_control.fAllowOtherInputs = true; - coin_control.Select(COutPoint(vCoins.at(0).tx->GetHash(), vCoins.at(0).i)); - BOOST_CHECK(testWallet.SelectCoins(vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb, bnb_used)); - BOOST_CHECK(!bnb_used); - BOOST_CHECK(!coin_selection_params_bnb.use_bnb); + add_coin(1 * CENT); + vCoins.at(0).nInputBytes = 40; + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + coin_selection_params_bnb.m_subtract_fee_outputs = true; + BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); + + // Make sure that can use BnB when there are preset inputs + empty_wallet(); + { + std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + bool firstRun; + wallet->LoadWallet(firstRun); + wallet->SetupLegacyScriptPubKeyMan(); + LOCK(wallet->cs_wallet); + add_coin(*wallet, 5 * CENT, 6 * 24, false, 0, true); + add_coin(*wallet, 3 * CENT, 6 * 24, false, 0, true); + add_coin(*wallet, 2 * CENT, 6 * 24, false, 0, true); + CCoinControl coin_control; + coin_control.fAllowOtherInputs = true; + coin_control.Select(COutPoint(vCoins.at(0).tx->GetHash(), vCoins.at(0).i)); + coin_selection_params_bnb.effective_fee = CFeeRate(0); + BOOST_CHECK(wallet->SelectCoins(vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(bnb_used); + BOOST_CHECK(coin_selection_params_bnb.use_bnb); + } } BOOST_AUTO_TEST_CASE(knapsack_solver_test) @@ -270,6 +301,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) bool bnb_used; LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) @@ -549,6 +581,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) bool bnb_used; LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); empty_wallet(); @@ -567,6 +600,8 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { + testWallet.SetupLegacyScriptPubKeyMan(); + // Random generator stuff std::default_random_engine generator; std::exponential_distribution<double> distribution (100); diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index c961456572..f4a4c9fa7c 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -7,7 +7,7 @@ #include <boost/test/unit_test.hpp> #include <fs.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/db.h> diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 86ba0013fe..797a0d634f 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index e2b7075085..6ba7d66b7c 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -6,7 +6,8 @@ #define BITCOIN_WALLET_TEST_INIT_TEST_FIXTURE_H #include <interfaces/chain.h> -#include <test/setup_common.h> +#include <node/context.h> +#include <test/util/setup_common.h> struct InitWalletDirTestingSetup: public BasicTestingSetup { @@ -17,7 +18,8 @@ struct InitWalletDirTestingSetup: public BasicTestingSetup { fs::path m_datadir; fs::path m_cwd; std::map<std::string, fs::path> m_walletdir_path_cases; - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + NodeContext m_node; + std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::ChainClient> m_chain_client; }; diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 279542ffad..c228e06009 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -5,7 +5,8 @@ #include <boost/test/unit_test.hpp> #include <noui.h> -#include <test/setup_common.h> +#include <test/util/logging.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <wallet/test/init_test_fixture.h> @@ -34,28 +35,31 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) { SetWalletDir(m_walletdir_path_cases["nonexistent"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("does not exist"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_directory) { SetWalletDir(m_walletdir_path_cases["file"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is not a directory"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative) { SetWalletDir(m_walletdir_path_cases["relative"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is a relative path"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 062fef7748..4c0e4dc653 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -3,9 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> +#include <node/context.h> #include <script/script.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/ismine.h> #include <wallet/wallet.h> @@ -26,7 +27,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CKey uncompressedKey; uncompressedKey.MakeNewKey(false); CPubKey uncompressedPubkey = uncompressedKey.GetPubKey(); - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(); + NodeContext node; + std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); CScript scriptPubKey; isminetype result; @@ -34,363 +36,383 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2PK uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2PKH compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2PKH uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2SH { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemScript = GetScriptForDestination(PKHash(pubkeys[0])); scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); // Keystore does not have redeemScript or key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has redeemScript but no key - BOOST_CHECK(keystore.AddCScript(redeemScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has redeemScript and key - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // (P2PKH inside) P2SH inside P2SH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemscript_inner = GetScriptForDestination(PKHash(pubkeys[0])); CScript redeemscript = GetScriptForDestination(ScriptHash(redeemscript_inner)); scriptPubKey = GetScriptForDestination(ScriptHash(redeemscript)); - BOOST_CHECK(keystore.AddCScript(redeemscript)); - BOOST_CHECK(keystore.AddCScript(redeemscript_inner)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemscript_inner)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // (P2PKH inside) P2SH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemscript = GetScriptForDestination(PKHash(pubkeys[0])); CScript witnessscript = GetScriptForDestination(ScriptHash(redeemscript)); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); - BOOST_CHECK(keystore.AddCScript(witnessscript)); - BOOST_CHECK(keystore.AddCScript(redeemscript)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // P2WPKH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessscript = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); - BOOST_CHECK(keystore.AddCScript(witnessscript)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // (P2PKH inside) P2WSH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessscript_inner = GetScriptForDestination(PKHash(pubkeys[0])); CScript witnessscript = GetScriptForDestination(WitnessV0ScriptHash(witnessscript_inner)); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); - BOOST_CHECK(keystore.AddCScript(witnessscript_inner)); - BOOST_CHECK(keystore.AddCScript(witnessscript)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript_inner)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // P2WPKH compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); // Keystore implicitly has key and P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2WPKH uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(uncompressedKey)); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(uncompressedPubkey))); // Keystore has key, but no P2SH redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key and P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // scriptPubKey multisig { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); // Keystore does not have any keys - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has 1/2 keys - BOOST_CHECK(keystore.AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has 2/2 keys - BOOST_CHECK(keystore.AddKey(keys[1])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has 2/2 keys and the script - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // P2SH multisig { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - BOOST_CHECK(keystore.AddKey(keys[1])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); CScript redeemScript = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); // Keystore has no redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has redeemScript - BOOST_CHECK(keystore.AddCScript(redeemScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2WSH multisig with compressed keys { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); - BOOST_CHECK(keystore.AddKey(keys[1])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); CScript witnessScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); // Keystore has keys, but no witnessScript or P2SH redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys and witnessScript, but no P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(witnessScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys, witnessScript, P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // P2WSH multisig with uncompressed key { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - BOOST_CHECK(keystore.AddKey(keys[1])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); CScript witnessScript = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); // Keystore has keys, but no witnessScript or P2SH redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys and witnessScript, but no P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(witnessScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys, witnessScript, P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // P2WSH multisig wrapped in P2SH { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); CScript redeemScript = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); // Keystore has no witnessScript, P2SH redeemScript, or keys - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has witnessScript and P2SH redeemScript, but no keys - BOOST_CHECK(keystore.AddCScript(redeemScript)); - BOOST_CHECK(keystore.AddCScript(witnessScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys, witnessScript, P2SH redeemScript - BOOST_CHECK(keystore.AddKey(keys[0])); - BOOST_CHECK(keystore.AddKey(keys[1])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } // OP_RETURN { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_RETURN << ToByteVector(pubkeys[0]); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // witness unspendable { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_0 << ToByteVector(ParseHex("aabb")); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // witness unknown { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_16 << ToByteVector(ParseHex("aabb")); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } // Nonstandard { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_9 << OP_ADD << OP_11 << OP_EQUAL; - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 0400f1207c..f923de6178 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -9,14 +9,15 @@ #include <wallet/wallet.h> #include <boost/test/unit_test.hpp> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/test/wallet_test_fixture.h> BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) BOOST_AUTO_TEST_CASE(psbt_updater_test) { - LOCK(m_wallet.cs_wallet); + auto spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(m_wallet.cs_wallet, spk_man->cs_KeyStore); // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); @@ -35,23 +36,23 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) CScript rs1; CDataStream s_rs1(ParseHex("475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae"), SER_NETWORK, PROTOCOL_VERSION); s_rs1 >> rs1; - m_wallet.AddCScript(rs1); + spk_man->AddCScript(rs1); CScript rs2; CDataStream s_rs2(ParseHex("2200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903"), SER_NETWORK, PROTOCOL_VERSION); s_rs2 >> rs2; - m_wallet.AddCScript(rs2); + spk_man->AddCScript(rs2); CScript ws1; CDataStream s_ws1(ParseHex("47522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae"), SER_NETWORK, PROTOCOL_VERSION); s_ws1 >> ws1; - m_wallet.AddCScript(ws1); + spk_man->AddCScript(ws1); // Add hd seed CKey key = DecodeSecret("5KSSJQ7UNfFGwVgpCZDSHm5rVNhMFcFtvWM3zQ8mW4qNDEN7LFd"); // Mainnet and uncompressed form of cUkG8i1RFfWGWy5ziR11zJ5V4U4W3viSFCfyJmZnvQaUsd1xuF3T - CPubKey master_pub_key = m_wallet.DeriveNewSeed(key); - m_wallet.SetHDSeed(master_pub_key); - m_wallet.NewKeyPool(); + CPubKey master_pub_key = spk_man->DeriveNewSeed(key); + spk_man->SetHDSeed(master_pub_key); + spk_man->NewKeyPool(); // Call FillPSBT PartiallySignedTransaction psbtx; @@ -67,6 +68,15 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) ssTx << psbtx; std::string final_hex = HexStr(ssTx.begin(), ssTx.end()); BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); + + // Mutate the transaction so that one of the inputs is invalid + psbtx.tx->vin[0].prevout.n = 2; + + // Try to sign the mutated input + SignatureData sigdata; + psbtx.inputs[0].FillSignatureData(sigdata); + const std::unique_ptr<SigningProvider> provider = m_wallet.GetSigningProvider(ws1, sigdata); + BOOST_CHECK(!SignPSBTInput(*provider, psbtx, 0, SIGHASH_ALL)); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 2f41813234..97f8c94fa6 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <wallet/crypter.h> diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index c1dbecdf8c..4e4129fb2c 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H #define BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <interfaces/chain.h> #include <interfaces/wallet.h> +#include <node/context.h> #include <wallet/wallet.h> #include <memory> @@ -18,7 +19,8 @@ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + NodeContext m_node; + std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); CWallet m_wallet; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a2b2a7b227..a487e9e2e0 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -9,9 +9,10 @@ #include <vector> #include <interfaces/chain.h> +#include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <validation.h> #include <wallet/coincontrol.h> #include <wallet/test/wallet_test_fixture.h> @@ -27,8 +28,9 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static void AddKey(CWallet& wallet, const CKey& key) { - LOCK(wallet.cs_wallet); - wallet.AddKeyPubKey(key, key.GetPubKey()); + auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); + spk_man->AddKeyPubKey(key, key.GetPubKey()); } BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) @@ -39,13 +41,18 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = ::ChainActive().Tip(); - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); // Verify ScanForWalletTransactions accommodates a null start block. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -61,6 +68,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // and new block files. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -80,6 +91,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // file. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -98,6 +113,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -118,7 +137,8 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = ::ChainActive().Tip(); - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); @@ -131,6 +151,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // after. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -185,7 +206,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); @@ -194,9 +216,10 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(wallet->cs_wallet); - wallet->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; - wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); + spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); JSONRPCRequest request; request.params.setArray(); @@ -210,6 +233,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // were scanned, and no prior blocks were scanned. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); JSONRPCRequest request; request.params.setArray(); @@ -239,26 +263,30 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); - LOCK(wallet.cs_wallet); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); - wtx.SetConf(CWalletTx::Status::CONFIRMED, ::ChainActive().Tip()->GetBlockHash(), 0); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 0); + wtx.m_confirm = confirm; // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(*locked_chain), 0); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); - wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(*locked_chain), 50*COIN); + BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); } static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) @@ -289,7 +317,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 wallet.AddToWallet(wtx); } if (block) { - wtx.SetConf(CWalletTx::Status::CONFIRMED, block->GetBlockHash(), 0); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block->nHeight, block->GetBlockHash(), 0); + wtx.m_confirm = confirm; } wallet.AddToWallet(wtx); return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; @@ -327,9 +356,10 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests) { CTxDestination dest = PKHash(); LOCK(m_wallet.cs_wallet); - m_wallet.AddDestData(dest, "misc", "val_misc"); - m_wallet.AddDestData(dest, "rr0", "val_rr0"); - m_wallet.AddDestData(dest, "rr1", "val_rr1"); + WalletBatch batch{m_wallet.GetDatabase()}; + m_wallet.AddDestData(batch, dest, "misc", "val_misc"); + m_wallet.AddDestData(batch, dest, "rr0", "val_rr0"); + m_wallet.AddDestData(batch, dest, "rr1", "val_rr1"); auto values = m_wallet.GetDestValues("rr"); BOOST_CHECK_EQUAL(values.size(), 2U); @@ -337,37 +367,37 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests) BOOST_CHECK_EQUAL(values[1], "val_rr1"); } -// Test some watch-only wallet methods by the procedure of loading (LoadWatchOnly), +// Test some watch-only LegacyScriptPubKeyMan methods by the procedure of loading (LoadWatchOnly), // checking (HaveWatchOnly), getting (GetWatchPubKey) and removing (RemoveWatchOnly) a // given PubKey, resp. its corresponding P2PK Script. Results of the the impact on // the address -> PubKey map is dependent on whether the PubKey is a point on the curve -static void TestWatchOnlyPubKey(CWallet& wallet, const CPubKey& add_pubkey) +static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& add_pubkey) { CScript p2pk = GetScriptForRawPubKey(add_pubkey); CKeyID add_address = add_pubkey.GetID(); CPubKey found_pubkey; - LOCK(wallet.cs_wallet); + LOCK(spk_man->cs_KeyStore); // all Scripts (i.e. also all PubKeys) are added to the general watch-only set - BOOST_CHECK(!wallet.HaveWatchOnly(p2pk)); - wallet.LoadWatchOnly(p2pk); - BOOST_CHECK(wallet.HaveWatchOnly(p2pk)); + BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); + spk_man->LoadWatchOnly(p2pk); + BOOST_CHECK(spk_man->HaveWatchOnly(p2pk)); // only PubKeys on the curve shall be added to the watch-only address -> PubKey map bool is_pubkey_fully_valid = add_pubkey.IsFullyValid(); if (is_pubkey_fully_valid) { - BOOST_CHECK(wallet.GetWatchPubKey(add_address, found_pubkey)); + BOOST_CHECK(spk_man->GetWatchPubKey(add_address, found_pubkey)); BOOST_CHECK(found_pubkey == add_pubkey); } else { - BOOST_CHECK(!wallet.GetWatchPubKey(add_address, found_pubkey)); + BOOST_CHECK(!spk_man->GetWatchPubKey(add_address, found_pubkey)); BOOST_CHECK(found_pubkey == CPubKey()); // passed key is unchanged } - wallet.RemoveWatchOnly(p2pk); - BOOST_CHECK(!wallet.HaveWatchOnly(p2pk)); + spk_man->RemoveWatchOnly(p2pk); + BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); if (is_pubkey_fully_valid) { - BOOST_CHECK(!wallet.GetWatchPubKey(add_address, found_pubkey)); + BOOST_CHECK(!spk_man->GetWatchPubKey(add_address, found_pubkey)); BOOST_CHECK(found_pubkey == add_pubkey); // passed key is unchanged } } @@ -382,37 +412,38 @@ static void PollutePubKey(CPubKey& pubkey) assert(pubkey.IsValid()); } -// Test watch-only wallet logic for PubKeys +// Test watch-only logic for PubKeys BOOST_AUTO_TEST_CASE(WatchOnlyPubKeys) { CKey key; CPubKey pubkey; + LegacyScriptPubKeyMan* spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); - BOOST_CHECK(!m_wallet.HaveWatchOnly()); + BOOST_CHECK(!spk_man->HaveWatchOnly()); // uncompressed valid PubKey key.MakeNewKey(false); pubkey = key.GetPubKey(); assert(!pubkey.IsCompressed()); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // uncompressed cryptographically invalid PubKey PollutePubKey(pubkey); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // compressed valid PubKey key.MakeNewKey(true); pubkey = key.GetPubKey(); assert(pubkey.IsCompressed()); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // compressed cryptographically invalid PubKey PollutePubKey(pubkey); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // invalid empty PubKey pubkey = CPubKey(); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); } class ListCoinsTestingSetup : public TestChain100Setup @@ -422,6 +453,10 @@ public: { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + { + LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); @@ -460,13 +495,16 @@ public: LOCK(cs_main); LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, ::ChainActive().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - it->second.SetConf(CWalletTx::Status::CONFIRMED, ::ChainActive().Tip()->GetBlockHash(), 1); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 1); + it->second.m_confirm = confirm; return it->second; } - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + NodeContext m_node; + std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<CWallet> wallet; }; @@ -538,8 +576,10 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 159d4f78c6..4a38571dfc 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,8 +45,9 @@ const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ static const size_t OUTPUT_GROUP_MAX_ENTRIES = 10; -static CCriticalSection cs_wallets; +static RecursiveMutex cs_wallets; static std::vector<std::shared_ptr<CWallet>> vpwallets GUARDED_BY(cs_wallets); +static std::list<LoadWalletFn> g_load_wallet_fns GUARDED_BY(cs_wallets); bool AddWallet(const std::shared_ptr<CWallet>& wallet) { @@ -55,6 +56,7 @@ bool AddWallet(const std::shared_ptr<CWallet>& wallet) std::vector<std::shared_ptr<CWallet>>::const_iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); if (i != vpwallets.end()) return false; vpwallets.push_back(wallet); + wallet->ConnectScriptPubKeyManNotifiers(); return true; } @@ -89,6 +91,13 @@ std::shared_ptr<CWallet> GetWallet(const std::string& name) return nullptr; } +std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) +{ + LOCK(cs_wallets); + auto it = g_load_wallet_fns.emplace(g_load_wallet_fns.end(), std::move(load_wallet)); + return interfaces::MakeHandler([it] { LOCK(cs_wallets); g_load_wallet_fns.erase(it); }); +} + static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; static std::set<std::string> g_unloading_wallet_set; @@ -210,9 +219,15 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& } // Set a seed for the wallet - CPubKey master_pub_key = wallet->GenerateNewSeed(); - wallet->SetHDSeed(master_pub_key); - wallet->NewKeyPool(); + { + LOCK(wallet->cs_wallet); + for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = "Unable to generate initial keys"; + return WalletCreationStatus::CREATION_FAILED; + } + } + } // Relock the wallet wallet->Lock(); @@ -224,9 +239,7 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& return WalletCreationStatus::SUCCESS; } -const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; - -const uint256 CWalletTx::ABANDON_HASH(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); +const uint256 CWalletTx::ABANDON_HASH(UINT256_ONE()); /** @defgroup mapWallet * @@ -238,18 +251,6 @@ std::string COutput::ToString() const return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider) -{ - std::vector<CScript> dummy; - FlatSigningProvider out; - InferDescriptor(spk, provider)->Expand(0, DUMMY_SIGNING_PROVIDER, dummy, out); - std::vector<CKeyID> ret; - for (const auto& entry : out.pubkeys) { - ret.push_back(entry.first); - } - return ret; -} - const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { LOCK(cs_wallet); @@ -259,354 +260,19 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const return &(it->second); } -CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) -{ - assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - assert(!IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); - AssertLockHeld(cs_wallet); - bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets - - CKey secret; - - // Create new metadata - int64_t nCreationTime = GetTime(); - CKeyMetadata metadata(nCreationTime); - - // use HD key derivation if HD was enabled during wallet creation and a seed is present - if (IsHDEnabled()) { - DeriveNewChildKey(batch, metadata, secret, (CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); - } else { - secret.MakeNewKey(fCompressed); - } - - // Compressed public keys were introduced in version 0.6.0 - if (fCompressed) { - SetMinVersion(FEATURE_COMPRPUBKEY); - } - - CPubKey pubkey = secret.GetPubKey(); - assert(secret.VerifyPubKey(pubkey)); - - mapKeyMetadata[pubkey.GetID()] = metadata; - UpdateTimeFirstKey(nCreationTime); - - if (!AddKeyPubKeyWithDB(batch, secret, pubkey)) { - throw std::runtime_error(std::string(__func__) + ": AddKey failed"); - } - return pubkey; -} - -void CWallet::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal) -{ - // for now we use a fixed keypath scheme of m/0'/0'/k - CKey seed; //seed (256bit) - CExtKey masterKey; //hd master key - CExtKey accountKey; //key at m/0' - CExtKey chainChildKey; //key at m/0'/0' (external) or m/0'/1' (internal) - CExtKey childKey; //key at m/0'/0'/<n>' - - // try to get the seed - if (!GetKey(hdChain.seed_id, seed)) - throw std::runtime_error(std::string(__func__) + ": seed not found"); - - masterKey.SetSeed(seed.begin(), seed.size()); - - // derive m/0' - // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) - masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); - - // derive m/0'/0' (external chain) OR m/0'/1' (internal chain) - assert(internal ? CanSupportFeature(FEATURE_HD_SPLIT) : true); - accountKey.Derive(chainChildKey, BIP32_HARDENED_KEY_LIMIT+(internal ? 1 : 0)); - - // derive child key at next index, skip keys already known to the wallet - do { - // always derive hardened keys - // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range - // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 - if (internal) { - chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; - metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nInternalChainCounter++; - } - else { - chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; - metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nExternalChainCounter++; - } - } while (HaveKey(childKey.key.GetPubKey().GetID())); - secret = childKey.key; - metadata.hd_seed_id = hdChain.seed_id; - CKeyID master_id = masterKey.key.GetPubKey().GetID(); - std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); - metadata.has_key_origin = true; - // update the chain model in the database - if (!batch.WriteHDChain(hdChain)) - throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); -} - -bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) -{ - AssertLockHeld(cs_wallet); - - // Make sure we aren't adding private keys to private key disabled wallets - assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - - // FillableSigningProvider has no concept of wallet databases, but calls AddCryptedKey - // which is overridden below. To avoid flushes, the database handle is - // tunneled through to it. - bool needsDB = !encrypted_batch; - if (needsDB) { - encrypted_batch = &batch; - } - if (!AddKeyPubKeyInner(secret, pubkey)) { - if (needsDB) encrypted_batch = nullptr; - return false; - } - if (needsDB) encrypted_batch = nullptr; - - // check if we need to remove from watch-only - CScript script; - script = GetScriptForDestination(PKHash(pubkey)); - if (HaveWatchOnly(script)) { - RemoveWatchOnly(script); - } - script = GetScriptForRawPubKey(pubkey); - if (HaveWatchOnly(script)) { - RemoveWatchOnly(script); - } - - if (!IsCrypted()) { - return batch.WriteKey(pubkey, - secret.GetPrivKey(), - mapKeyMetadata[pubkey.GetID()]); - } - UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); - return true; -} - -bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) -{ - WalletBatch batch(*database); - return CWallet::AddKeyPubKeyWithDB(batch, secret, pubkey); -} - -bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, - const std::vector<unsigned char> &vchCryptedSecret) -{ - if (!AddCryptedKeyInner(vchPubKey, vchCryptedSecret)) - return false; - { - LOCK(cs_wallet); - if (encrypted_batch) - return encrypted_batch->WriteCryptedKey(vchPubKey, - vchCryptedSecret, - mapKeyMetadata[vchPubKey.GetID()]); - else - return WalletBatch(*database).WriteCryptedKey(vchPubKey, - vchCryptedSecret, - mapKeyMetadata[vchPubKey.GetID()]); - } -} - -void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) -{ - AssertLockHeld(cs_wallet); - UpdateTimeFirstKey(meta.nCreateTime); - mapKeyMetadata[keyID] = meta; -} - -void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) -{ - AssertLockHeld(cs_wallet); - UpdateTimeFirstKey(meta.nCreateTime); - m_script_metadata[script_id] = meta; -} - void CWallet::UpgradeKeyMetadata() { - AssertLockHeld(cs_wallet); if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { return; } - std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database); - for (auto& meta_pair : mapKeyMetadata) { - CKeyMetadata& meta = meta_pair.second; - if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin - CKey key; - GetKey(meta.hd_seed_id, key); - CExtKey masterKey; - masterKey.SetSeed(key.begin(), key.size()); - // Add to map - CKeyID master_id = masterKey.key.GetPubKey().GetID(); - std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); - if (!ParseHDKeypath(meta.hdKeypath, meta.key_origin.path)) { - throw std::runtime_error("Invalid stored hdKeypath"); - } - meta.has_key_origin = true; - if (meta.nVersion < CKeyMetadata::VERSION_WITH_KEY_ORIGIN) { - meta.nVersion = CKeyMetadata::VERSION_WITH_KEY_ORIGIN; - } - - // Write meta to wallet - CPubKey pubkey; - if (GetPubKey(meta_pair.first, pubkey)) { - batch->WriteKeyMetadata(meta, pubkey, true); - } - } - } - batch.reset(); //write before setting the flag - SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); -} - -bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) -{ - return AddCryptedKeyInner(vchPubKey, vchCryptedSecret); -} - -/** - * Update wallet first key creation time. This should be called whenever keys - * are added to the wallet, with the oldest key creation time. - */ -void CWallet::UpdateTimeFirstKey(int64_t nCreateTime) -{ - AssertLockHeld(cs_wallet); - if (nCreateTime <= 1) { - // Cannot determine birthday information, so set the wallet birthday to - // the beginning of time. - nTimeFirstKey = 1; - } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { - nTimeFirstKey = nCreateTime; - } -} - -bool CWallet::AddCScript(const CScript& redeemScript) -{ - WalletBatch batch(*database); - return AddCScriptWithDB(batch, redeemScript); -} - -bool CWallet::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript) -{ - if (!FillableSigningProvider::AddCScript(redeemScript)) - return false; - if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) { - UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); - return true; - } - return false; -} - -bool CWallet::LoadCScript(const CScript& redeemScript) -{ - /* A sanity check was added in pull #3843 to avoid adding redeemScripts - * that never can be redeemed. However, old wallets may still contain - * these. Do not add them to the wallet and warn. */ - if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) - { - std::string strAddr = EncodeDestination(ScriptHash(redeemScript)); - WalletLogPrintf("%s: Warning: This wallet contains a redeemScript of size %i which exceeds maximum size %i thus can never be redeemed. Do not use address %s.\n", __func__, redeemScript.size(), MAX_SCRIPT_ELEMENT_SIZE, strAddr); - return true; - } - - return FillableSigningProvider::AddCScript(redeemScript); -} - -static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) -{ - std::vector<std::vector<unsigned char>> solutions; - return Solver(dest, solutions) == TX_PUBKEY && - (pubKeyOut = CPubKey(solutions[0])).IsFullyValid(); -} - -bool CWallet::AddWatchOnlyInMem(const CScript &dest) -{ - LOCK(cs_KeyStore); - setWatchOnly.insert(dest); - CPubKey pubKey; - if (ExtractPubKey(dest, pubKey)) { - mapWatchKeys[pubKey.GetID()] = pubKey; - ImplicitlyLearnRelatedKeyScripts(pubKey); - } - return true; -} - -bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) -{ - if (!AddWatchOnlyInMem(dest)) - return false; - const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; - UpdateTimeFirstKey(meta.nCreateTime); - NotifyWatchonlyChanged(true); - if (batch.WriteWatchOnly(dest, meta)) { - UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); - return true; - } - return false; -} - -bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) -{ - m_script_metadata[CScriptID(dest)].nCreateTime = create_time; - return AddWatchOnlyWithDB(batch, dest); -} - -bool CWallet::AddWatchOnly(const CScript& dest) -{ - WalletBatch batch(*database); - return AddWatchOnlyWithDB(batch, dest); -} - -bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) -{ - m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; - return AddWatchOnly(dest); -} - -bool CWallet::RemoveWatchOnly(const CScript &dest) -{ - AssertLockHeld(cs_wallet); - { - LOCK(cs_KeyStore); - setWatchOnly.erase(dest); - CPubKey pubKey; - if (ExtractPubKey(dest, pubKey)) { - mapWatchKeys.erase(pubKey.GetID()); - } - // Related CScripts are not removed; having superfluous scripts around is - // harmless (see comment in ImplicitlyLearnRelatedKeyScripts). + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return; } - if (!HaveWatchOnly()) - NotifyWatchonlyChanged(false); - if (!WalletBatch(*database).EraseWatchOnly(dest)) - return false; - - return true; -} - -bool CWallet::LoadWatchOnly(const CScript &dest) -{ - return AddWatchOnlyInMem(dest); -} - -bool CWallet::HaveWatchOnly(const CScript &dest) const -{ - LOCK(cs_KeyStore); - return setWatchOnly.count(dest) > 0; -} - -bool CWallet::HaveWatchOnly() const -{ - LOCK(cs_KeyStore); - return (!setWatchOnly.empty()); + spk_man->UpgradeKeyMetadata(); + SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_keys) @@ -798,7 +464,7 @@ void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> ran * Outpoint is spent if any non-conflicted transaction * spends it: */ -bool CWallet::IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash, unsigned int n) const +bool CWallet::IsSpent(const uint256& hash, unsigned int n) const { const COutPoint outpoint(hash, n); std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range; @@ -809,7 +475,7 @@ bool CWallet::IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash const uint256& wtxid = it->second; std::map<uint256, CWalletTx>::const_iterator mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { - int depth = mit->second.GetDepthInMainChain(locked_chain); + int depth = mit->second.GetDepthInMainChain(); if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) return true; // Spent } @@ -878,8 +544,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) { LOCK(cs_wallet); mapMasterKeys[++nMasterKeyMaxID] = kMasterKey; - assert(!encrypted_batch); - encrypted_batch = new WalletBatch(*database); + WalletBatch* encrypted_batch = new WalletBatch(*database); if (!encrypted_batch->TxnBegin()) { delete encrypted_batch; encrypted_batch = nullptr; @@ -887,14 +552,16 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) } encrypted_batch->WriteMasterKey(nMasterKeyMaxID, kMasterKey); - if (!EncryptKeys(_vMasterKey)) - { - encrypted_batch->TxnAbort(); - delete encrypted_batch; - encrypted_batch = nullptr; - // We now probably have half of our keys encrypted in memory, and half not... - // die and let the user reload the unencrypted wallet. - assert(false); + for (const auto& spk_man_pair : m_spk_managers) { + auto spk_man = spk_man_pair.second.get(); + if (!spk_man->Encrypt(_vMasterKey, encrypted_batch)) { + encrypted_batch->TxnAbort(); + delete encrypted_batch; + encrypted_batch = nullptr; + // We now probably have half of our keys encrypted in memory, and half not... + // die and let the user reload the unencrypted wallet. + assert(false); + } } // Encryption was introduced in version 0.4.0 @@ -915,11 +582,13 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Unlock(strWalletPassphrase); // if we are using HD, replace the HD seed with a new one - if (IsHDEnabled()) { - SetHDSeed(GenerateNewSeed()); + if (auto spk_man = GetLegacyScriptPubKeyMan()) { + if (spk_man->IsHDEnabled()) { + if (!spk_man->SetupGeneration(true)) { + return false; + } + } } - - NewKeyPool(); Lock(); // Need to completely rewrite the wallet file; if we don't, bdb might keep @@ -1044,35 +713,53 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) return success; } -void CWallet::SetUsedDestinationState(const uint256& hash, unsigned int n, bool used) +void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) { + AssertLockHeld(cs_wallet); const CWalletTx* srctx = GetWalletTx(hash); if (!srctx) return; CTxDestination dst; if (ExtractDestination(srctx->tx->vout[n].scriptPubKey, dst)) { - if (::IsMine(*this, dst)) { - LOCK(cs_wallet); + if (IsMine(dst)) { if (used && !GetDestData(dst, "used", nullptr)) { - AddDestData(dst, "used", "p"); // p for "present", opposite of absent (null) + if (AddDestData(batch, dst, "used", "p")) { // p for "present", opposite of absent (null) + tx_destinations.insert(dst); + } } else if (!used && GetDestData(dst, "used", nullptr)) { - EraseDestData(dst, "used"); + EraseDestData(batch, dst, "used"); } } } } -bool CWallet::IsUsedDestination(const CTxDestination& dst) const -{ - LOCK(cs_wallet); - return ::IsMine(*this, dst) && GetDestData(dst, "used", nullptr); -} - bool CWallet::IsUsedDestination(const uint256& hash, unsigned int n) const { + AssertLockHeld(cs_wallet); CTxDestination dst; const CWalletTx* srctx = GetWalletTx(hash); - return srctx && ExtractDestination(srctx->tx->vout[n].scriptPubKey, dst) && IsUsedDestination(dst); + if (srctx) { + assert(srctx->tx->vout.size() > n); + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + // When descriptor wallets arrive, these additional checks are + // likely superfluous and can be optimized out + assert(spk_man != nullptr); + for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { + WitnessV0KeyHash wpkh_dest(keyid); + if (GetDestData(wpkh_dest, "used", nullptr)) { + return true; + } + ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); + if (GetDestData(sh_wpkh_dest, "used", nullptr)) { + return true; + } + PKHash pkh_dest(keyid); + if (GetDestData(pkh_dest, "used", nullptr)) { + return true; + } + } + } + return false; } bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) @@ -1085,10 +772,14 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) if (IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)) { // Mark used destinations + std::set<CTxDestination> tx_destinations; + for (const CTxIn& txin : wtxIn.tx->vin) { const COutPoint& op = txin.prevout; - SetUsedDestinationState(op.hash, op.n, true); + SetUsedDestinationState(batch, op.hash, op.n, true, tx_destinations); } + + MarkDestinationsDirty(tx_destinations); } // Inserts only if not already there, returns tx inserted or tx found @@ -1111,10 +802,12 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) wtx.m_confirm.status = wtxIn.m_confirm.status; wtx.m_confirm.nIndex = wtxIn.m_confirm.nIndex; wtx.m_confirm.hashBlock = wtxIn.m_confirm.hashBlock; + wtx.m_confirm.block_height = wtxIn.m_confirm.block_height; fUpdated = true; } else { assert(wtx.m_confirm.nIndex == wtxIn.m_confirm.nIndex); assert(wtx.m_confirm.hashBlock == wtxIn.m_confirm.hashBlock); + assert(wtx.m_confirm.block_height == wtxIn.m_confirm.block_height); } if (wtxIn.fFromMe && wtxIn.fFromMe != wtx.fFromMe) { @@ -1163,14 +856,24 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) void CWallet::LoadToWallet(CWalletTx& wtxIn) { - // If wallet doesn't have a chain (e.g wallet-tool), lock can't be taken. + // If wallet doesn't have a chain (e.g bitcoin-wallet), lock can't be taken. auto locked_chain = LockChain(); - // If tx hasn't been reorged out of chain while wallet being shutdown - // change tx status to UNCONFIRMED and reset hashBlock/nIndex. - if (!wtxIn.m_confirm.hashBlock.IsNull()) { - if (locked_chain && !locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock)) { + if (locked_chain) { + Optional<int> block_height = locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock); + if (block_height) { + // Update cached block height variable since it not stored in the + // serialized transaction. + wtxIn.m_confirm.block_height = *block_height; + } else if (wtxIn.isConflicted() || wtxIn.isConfirmed()) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. wtxIn.setUnconfirmed(); wtxIn.m_confirm.hashBlock = uint256(); + wtxIn.m_confirm.block_height = 0; wtxIn.m_confirm.nIndex = 0; } } @@ -1187,25 +890,25 @@ void CWallet::LoadToWallet(CWalletTx& wtxIn) if (it != mapWallet.end()) { CWalletTx& prevtx = it->second; if (prevtx.isConflicted()) { - MarkConflicted(prevtx.m_confirm.hashBlock, wtx.GetHash()); + MarkConflicted(prevtx.m_confirm.hashBlock, prevtx.m_confirm.block_height, wtx.GetHash()); } } } } -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate) { const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (!block_hash.IsNull()) { + if (!confirm.hashBlock.IsNull()) { for (const CTxIn& txin : tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { if (range.first->second != tx.GetHash()) { - WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), block_hash.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); - MarkConflicted(block_hash, range.first->second); + WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), confirm.hashBlock.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); + MarkConflicted(confirm.hashBlock, confirm.block_height, range.first->second); } range.first++; } @@ -1224,17 +927,8 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::St // loop though all outputs for (const CTxOut& txout: tx.vout) { - // extract addresses and check if they match with an unused keypool key - for (const auto& keyid : GetAffectedKeys(txout.scriptPubKey, *this)) { - std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); - if (mi != m_pool_key_to_index.end()) { - WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); - MarkReserveKeysAsUsed(mi->second); - - if (!TopUpKeyPool()) { - WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); - } - } + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->MarkUnusedAddresses(txout.scriptPubKey); } } @@ -1242,7 +936,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::St // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - wtx.SetConf(status, block_hash, posInBlock); + wtx.m_confirm = confirm; return AddToWallet(wtx, false); } @@ -1255,7 +949,7 @@ bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const auto locked_chain = chain().lock(); LOCK(cs_wallet); const CWalletTx* wtx = GetWalletTx(hashTx); - return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain(*locked_chain) == 0 && !wtx->InMempool(); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() == 0 && !wtx->InMempool(); } void CWallet::MarkInputsDirty(const CTransactionRef& tx) @@ -1268,9 +962,9 @@ void CWallet::MarkInputsDirty(const CTransactionRef& tx) } } -bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx) +bool CWallet::AbandonTransaction(const uint256& hashTx) { - auto locked_chain_recursive = chain().lock(); // Temporary. Removed in upcoming lock cleanup + auto locked_chain = chain().lock(); // Temporary. Removed in upcoming lock cleanup LOCK(cs_wallet); WalletBatch batch(*database, "r+"); @@ -1282,7 +976,7 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); CWalletTx& origtx = it->second; - if (origtx.GetDepthInMainChain(locked_chain) != 0 || origtx.InMempool()) { + if (origtx.GetDepthInMainChain() != 0 || origtx.InMempool()) { return false; } @@ -1295,14 +989,13 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(locked_chain); + int currentconfirm = wtx.GetDepthInMainChain(); // If the orig tx was not in block, none of its spends can be assert(currentconfirm <= 0); // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} if (currentconfirm == 0 && !wtx.isAbandoned()) { // If the orig tx was not in block/mempool, none of its spends can be in mempool assert(!wtx.InMempool()); - wtx.m_confirm.nIndex = 0; wtx.setAbandoned(); wtx.MarkDirty(); batch.WriteTx(wtx); @@ -1324,12 +1017,12 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui return true; } -void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) +void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - int conflictconfirms = -locked_chain->getBlockDepth(hashBlock); + int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; // If number of conflict confirms cannot be determined, this means // that the block is still unknown or not yet part of the main chain, // for example when loading the wallet during a reindex. Do nothing in that @@ -1352,12 +1045,13 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(*locked_chain); + int currentconfirm = wtx.GetDepthInMainChain(); if (conflictconfirms < currentconfirm) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. wtx.m_confirm.nIndex = 0; wtx.m_confirm.hashBlock = hashBlock; + wtx.m_confirm.block_height = conflicting_height; wtx.setConflicted(); wtx.MarkDirty(); batch.WriteTx(wtx); @@ -1376,9 +1070,9 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool update_tx) +void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx) { - if (!AddToWalletIfInvolvingMe(ptx, status, block_hash, posInBlock, update_tx)) + if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1390,7 +1084,8 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Status stat void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - SyncTransaction(ptx, CWalletTx::Status::UNCONFIRMED, {} /* block hash */, 0 /* position in block */); + CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); + SyncTransaction(ptx, confirm); auto it = mapWallet.find(ptx->GetHash()); if (it != mapWallet.end()) { @@ -1406,23 +1101,26 @@ void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { } } -void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) { +void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted, int height) +{ const uint256& block_hash = block.GetHash(); auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (size_t i = 0; i < block.vtx.size(); i++) { - SyncTransaction(block.vtx[i], CWalletTx::Status::CONFIRMED, block_hash, i); - TransactionRemovedFromMempool(block.vtx[i]); + m_last_block_processed_height = height; + m_last_block_processed = block_hash; + for (size_t index = 0; index < block.vtx.size(); index++) { + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, block_hash, index); + SyncTransaction(block.vtx[index], confirm); + TransactionRemovedFromMempool(block.vtx[index]); } for (const CTransactionRef& ptx : vtxConflicted) { TransactionRemovedFromMempool(ptx); } - - m_last_block_processed = block_hash; } -void CWallet::BlockDisconnected(const CBlock& block) { +void CWallet::BlockDisconnected(const CBlock& block, int height) +{ auto locked_chain = chain().lock(); LOCK(cs_wallet); @@ -1430,8 +1128,11 @@ void CWallet::BlockDisconnected(const CBlock& block) { // be unconfirmed, whether or not the transaction is added back to the mempool. // User may have to call abandontransaction again. It may be addressed in the // future with a stickier abandoned state or even removing abandontransaction call. + m_last_block_processed_height = height - 1; + m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - SyncTransaction(ptx, CWalletTx::Status::UNCONFIRMED, {} /* block hash */, 0 /* position in block */); + CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); + SyncTransaction(ptx, confirm); } } @@ -1448,7 +1149,7 @@ void CWallet::BlockUntilSyncedToCurrentChain() { // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); - chain().waitForNotificationsIfNewBlocksConnected(last_block_hash); + chain().waitForNotificationsIfTipChanged(last_block_hash); } @@ -1487,7 +1188,21 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const isminetype CWallet::IsMine(const CTxOut& txout) const { - return ::IsMine(*this, txout.scriptPubKey); + return IsMine(txout.scriptPubKey); +} + +isminetype CWallet::IsMine(const CTxDestination& dest) const +{ + return IsMine(GetScriptForDestination(dest)); +} + +isminetype CWallet::IsMine(const CScript& script) const +{ + isminetype result = ISMINE_NO; + for (const auto& spk_man_pair : m_spk_managers) { + result = std::max(result, spk_man_pair.second->IsMine(script)); + } + return result; } CAmount CWallet::GetCredit(const CTxOut& txout, const isminefilter& filter) const @@ -1511,7 +1226,7 @@ bool CWallet::IsChange(const CScript& script) const // a better way of identifying which outputs are 'the send' and which are // 'the change' will need to be implemented (maybe extend CWalletTx to remember // which output, if any, was change). - if (::IsMine(*this, script)) + if (IsMine(script)) { CTxDestination address; if (!ExtractDestination(script, address)) @@ -1601,92 +1316,26 @@ CAmount CWallet::GetChange(const CTransaction& tx) const return nChange; } -CPubKey CWallet::GenerateNewSeed() -{ - assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - CKey key; - key.MakeNewKey(true); - return DeriveNewSeed(key); -} - -CPubKey CWallet::DeriveNewSeed(const CKey& key) -{ - int64_t nCreationTime = GetTime(); - CKeyMetadata metadata(nCreationTime); - - // calculate the seed - CPubKey seed = key.GetPubKey(); - assert(key.VerifyPubKey(seed)); - - // set the hd keypath to "s" -> Seed, refers the seed to itself - metadata.hdKeypath = "s"; - metadata.has_key_origin = false; - metadata.hd_seed_id = seed.GetID(); - - { - LOCK(cs_wallet); - - // mem store the metadata - mapKeyMetadata[seed.GetID()] = metadata; - - // write the key&metadata to the database - if (!AddKeyPubKey(key, seed)) - throw std::runtime_error(std::string(__func__) + ": AddKeyPubKey failed"); - } - - return seed; -} - -void CWallet::SetHDSeed(const CPubKey& seed) -{ - LOCK(cs_wallet); - // store the keyid (hash160) together with - // the child index counter in the database - // as a hdchain object - CHDChain newHdChain; - newHdChain.nVersion = CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; - newHdChain.seed_id = seed.GetID(); - SetHDChain(newHdChain, false); - NotifyCanGetAddressesChanged(); - UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); -} - -void CWallet::SetHDChain(const CHDChain& chain, bool memonly) -{ - LOCK(cs_wallet); - if (!memonly && !WalletBatch(*database).WriteHDChain(chain)) - throw std::runtime_error(std::string(__func__) + ": writing chain failed"); - - hdChain = chain; -} - bool CWallet::IsHDEnabled() const { - return !hdChain.seed_id.IsNull(); -} - -bool CWallet::CanGenerateKeys() -{ - // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) - LOCK(cs_wallet); - return IsHDEnabled() || !CanSupportFeature(FEATURE_HD); + bool result = true; + for (const auto& spk_man_pair : m_spk_managers) { + result &= spk_man_pair.second->IsHDEnabled(); + } + return result; } bool CWallet::CanGetAddresses(bool internal) { LOCK(cs_wallet); - // Check if the keypool has keys - bool keypool_has_keys; - if (internal && CanSupportFeature(FEATURE_HD_SPLIT)) { - keypool_has_keys = setInternalKeyPool.size() > 0; - } else { - keypool_has_keys = KeypoolCountExternalKeys() > 0; - } - // If the keypool doesn't have keys, check if we can generate them - if (!keypool_has_keys) { - return CanGenerateKeys(); + if (m_spk_managers.empty()) return false; + for (OutputType t : OUTPUT_TYPES) { + auto spk_man = GetScriptPubKeyMan(t, internal); + if (spk_man && spk_man->CanGetAddresses(internal)) { + return true; + } } - return keypool_has_keys; + return false; } void CWallet::SetWalletFlag(uint64_t flags) @@ -1711,6 +1360,11 @@ void CWallet::UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } +void CWallet::UnsetBlankWalletFlag(WalletBatch& batch) +{ + UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); +} + bool CWallet::IsWalletFlagSet(uint64_t flag) const { return (m_wallet_flags & flag); @@ -1745,7 +1399,13 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(*this, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); + if (!provider) { + // We don't know about this scriptpbuKey; + return false; + } + + if (!ProduceSignature(*provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } UpdateInput(tx_in, sigdata); @@ -1770,97 +1430,53 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> bool CWallet::ImportScripts(const std::set<CScript> scripts, int64_t timestamp) { - WalletBatch batch(*database); - for (const auto& entry : scripts) { - CScriptID id(entry); - if (HaveCScript(id)) { - WalletLogPrintf("Already have script %s, skipping\n", HexStr(entry)); - continue; - } - if (!AddCScriptWithDB(batch, entry)) { - return false; - } - - if (timestamp > 0) { - m_script_metadata[CScriptID(entry)].nCreateTime = timestamp; - } - } - if (timestamp > 0) { - UpdateTimeFirstKey(timestamp); + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; } - - return true; + LOCK(spk_man->cs_KeyStore); + return spk_man->ImportScripts(scripts, timestamp); } bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) { - WalletBatch batch(*database); - for (const auto& entry : privkey_map) { - const CKey& key = entry.second; - CPubKey pubkey = key.GetPubKey(); - const CKeyID& id = entry.first; - assert(key.VerifyPubKey(pubkey)); - // Skip if we already have the key - if (HaveKey(id)) { - WalletLogPrintf("Already have key with pubkey %s, skipping\n", HexStr(pubkey)); - continue; - } - mapKeyMetadata[id].nCreateTime = timestamp; - // If the private key is not present in the wallet, insert it. - if (!AddKeyPubKeyWithDB(batch, key, pubkey)) { - return false; - } - UpdateTimeFirstKey(timestamp); + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; } - return true; + LOCK(spk_man->cs_KeyStore); + return spk_man->ImportPrivKeys(privkey_map, timestamp); } bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) { - WalletBatch batch(*database); - for (const auto& entry : key_origins) { - AddKeyOriginWithDB(batch, entry.second.first, entry.second.second); - } - for (const CKeyID& id : ordered_pubkeys) { - auto entry = pubkey_map.find(id); - if (entry == pubkey_map.end()) { - continue; - } - const CPubKey& pubkey = entry->second; - CPubKey temp; - if (GetPubKey(id, temp)) { - // Already have pubkey, skipping - WalletLogPrintf("Already have pubkey %s, skipping\n", HexStr(temp)); - continue; - } - if (!AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) { - return false; - } - mapKeyMetadata[id].nCreateTime = timestamp; - - // Add to keypool only works with pubkeys - if (add_keypool) { - AddKeypoolPubkeyWithDB(pubkey, internal, batch); - NotifyCanGetAddressesChanged(); - } + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; } - return true; + LOCK(spk_man->cs_KeyStore); + return spk_man->ImportPubKeys(ordered_pubkeys, pubkey_map, key_origins, add_keypool, internal, timestamp); } bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) { - WalletBatch batch(*database); - for (const CScript& script : script_pub_keys) { - if (!have_solving_data || !::IsMine(*this, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated - if (!AddWatchOnlyWithDB(batch, script, timestamp)) { - return false; + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; + } + LOCK(spk_man->cs_KeyStore); + if (!spk_man->ImportScriptPubKeys(script_pub_keys, have_solving_data, timestamp)) { + return false; + } + if (apply_label) { + WalletBatch batch(*database); + for (const CScript& script : script_pub_keys) { + CTxDestination dest; + ExtractDestination(script, dest); + if (IsValidDestination(dest)) { + SetAddressBookWithDB(batch, dest, label, "receive"); } } - CTxDestination dest; - ExtractDestination(script, dest); - if (apply_label && IsValidDestination(dest)) { - SetAddressBookWithDB(batch, dest, label, "receive"); - } } return true; } @@ -1868,11 +1484,9 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig) { std::vector<CTxOut> txouts; - // Look up the inputs. We should have already checked that this transaction - // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our - // wallet, with a valid index into the vout array, and the ability to sign. for (const CTxIn& input : tx.vin) { const auto mi = wallet->mapWallet.find(input.prevout.hash); + // Can not estimate size without knowing the input details if (mi == wallet->mapWallet.end()) { return -1; } @@ -1887,8 +1501,6 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wall { CMutableTransaction txNew(tx); if (!wallet->DummySignTx(txNew, txouts, use_max_sig)) { - // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) - // implies that we can sign for every input. return -1; } return GetVirtualTransactionSize(CTransaction(txNew)); @@ -2068,7 +1680,8 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], CWalletTx::Status::CONFIRMED, block_hash, posInBlock, fUpdate); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *block_height, block_hash, posInBlock); + SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -2116,7 +1729,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc return result; } -void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) +void CWallet::ReacceptWalletTransactions() { // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) @@ -2129,7 +1742,7 @@ void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -2140,11 +1753,11 @@ void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); std::string unused_err_string; - wtx.SubmitMemoryPoolAndRelay(unused_err_string, false, locked_chain); + wtx.SubmitMemoryPoolAndRelay(unused_err_string, false); } } -bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, interfaces::Chain::Lock& locked_chain) +bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay) { // Can't relay if wallet is not broadcasting if (!pwallet->GetBroadcastTransactions()) return false; @@ -2154,7 +1767,7 @@ bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, in // cause log spam. if (IsCoinBase()) return false; // Don't try to submit conflicted or confirmed transactions. - if (GetDepthInMainChain(locked_chain) != 0) return false; + if (GetDepthInMainChain() != 0) return false; // Submit transaction to mempool for relay pwallet->WalletLogPrintf("Submitting wtx %s to mempool for relay\n", GetHash().ToString()); @@ -2189,6 +1802,7 @@ CAmount CWalletTx::GetCachableAmount(AmountType type, const isminefilter& filter auto& amount = m_amounts[type]; if (recalculate || !amount.m_cached[filter]) { amount.Set(filter, type == DEBIT ? pwallet->GetDebit(*tx, filter) : pwallet->GetCredit(*tx, filter)); + m_is_cache_empty = false; } return amount.m_value[filter]; } @@ -2208,10 +1822,10 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const return debit; } -CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const +CAmount CWalletTx::GetCredit(const isminefilter& filter) const { // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase(locked_chain)) + if (IsImmatureCoinBase()) return 0; CAmount credit = 0; @@ -2225,16 +1839,16 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine return credit; } -CAmount CWalletTx::GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache) const +CAmount CWalletTx::GetImmatureCredit(bool fUseCache) const { - if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { + if (IsImmatureCoinBase() && IsInMainChain()) { return GetCachableAmount(IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); } return 0; } -CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache, const isminefilter& filter) const +CAmount CWalletTx::GetAvailableCredit(bool fUseCache, const isminefilter& filter) const { if (pwallet == nullptr) return 0; @@ -2243,7 +1857,7 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo bool allow_cache = (filter & ISMINE_ALL) && (filter & ISMINE_ALL) != ISMINE_ALL; // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase(locked_chain)) + if (IsImmatureCoinBase()) return 0; if (fUseCache && allow_cache && m_amounts[AVAILABLE_CREDIT].m_cached[filter]) { @@ -2255,7 +1869,7 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo uint256 hashTx = GetHash(); for (unsigned int i = 0; i < tx->vout.size(); i++) { - if (!pwallet->IsSpent(locked_chain, hashTx, i) && (allow_used_addresses || !pwallet->IsUsedDestination(hashTx, i))) { + if (!pwallet->IsSpent(hashTx, i) && (allow_used_addresses || !pwallet->IsUsedDestination(hashTx, i))) { const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, filter); if (!MoneyRange(nCredit)) @@ -2265,14 +1879,15 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo if (allow_cache) { m_amounts[AVAILABLE_CREDIT].Set(filter, nCredit); + m_is_cache_empty = false; } return nCredit; } -CAmount CWalletTx::GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache) const +CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool fUseCache) const { - if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { + if (IsImmatureCoinBase() && IsInMainChain()) { return GetCachableAmount(IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); } @@ -2295,32 +1910,37 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { + std::set<uint256> s; + return IsTrusted(locked_chain, s); +} + +bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const +{ // Quick answer in most cases - if (!locked_chain.checkFinalTx(*tx)) { - return false; - } - int nDepth = GetDepthInMainChain(locked_chain); - if (nDepth >= 1) - return true; - if (nDepth < 0) - return false; - if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) // using wtx's cached debit - return false; + if (!locked_chain.checkFinalTx(*tx)) return false; + int nDepth = GetDepthInMainChain(); + if (nDepth >= 1) return true; + if (nDepth < 0) return false; + // using wtx's cached debit + if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) return false; // Don't trust unconfirmed transactions from us unless they are in the mempool. - if (!InMempool()) - return false; + if (!InMempool()) return false; // Trusted if all inputs are from us and are in the mempool: for (const CTxIn& txin : tx->vin) { // Transactions not sent by us: not trusted const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); - if (parent == nullptr) - return false; + if (parent == nullptr) return false; const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; - if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) - return false; + // Check that this specific input being spent is trusted + if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; + // If we've already trusted this parent, continue + if (trusted_parents.count(parent->GetHash())) continue; + // Recurse to check that the parent is also trusted + if (!parent->IsTrusted(locked_chain, trusted_parents)) return false; + trusted_parents.insert(parent->GetHash()); } return true; } @@ -2374,7 +1994,7 @@ void CWallet::ResendWalletTransactions() // any confirmed or conflicting txs. if (wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; std::string unused_err_string; - if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true, *locked_chain)) ++submitted_tx_count; + if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true)) ++submitted_tx_count; } } // locked_chain and cs_wallet @@ -2406,13 +2026,14 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons { auto locked_chain = chain().lock(); LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(*locked_chain)}; - const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; - const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; + const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; + const int tx_depth{wtx.GetDepthInMainChain()}; + const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; @@ -2421,8 +2042,8 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += wtx.GetImmatureCredit(*locked_chain); - ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(*locked_chain); + ret.m_mine_immature += wtx.GetImmatureCredit(); + ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(); } } return ret; @@ -2456,6 +2077,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const int min_depth = {coinControl ? coinControl->m_min_depth : DEFAULT_MIN_DEPTH}; const int max_depth = {coinControl ? coinControl->m_max_depth : DEFAULT_MAX_DEPTH}; + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; @@ -2465,10 +2087,10 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - if (wtx.IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase()) continue; - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < 0) continue; @@ -2477,7 +2099,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain, trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -2528,7 +2150,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (IsLockedCoin(entry.first, i)) continue; - if (IsSpent(locked_chain, wtxid, i)) + if (IsSpent(wtxid, i)) continue; isminetype mine = IsMine(wtx.tx->vout[i]); @@ -2541,7 +2163,9 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - bool solvable = IsSolvable(*this, wtx.tx->vout[i].scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(wtx.tx->vout[i].scriptPubKey); + + bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); @@ -2574,7 +2198,7 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Ch for (const COutput& coin : availableCoins) { CTxDestination address; - if (coin.fSpendable && + if ((coin.fSpendable || (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.fSolvable)) && ExtractDestination(FindNonChangeParentOutput(*coin.tx->tx, coin.i).scriptPubKey, address)) { result[address].emplace_back(std::move(coin)); } @@ -2582,12 +2206,16 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Ch std::vector<COutPoint> lockedCoins; ListLockedCoins(lockedCoins); + // Include watch-only for LegacyScriptPubKeyMan wallets without private keys + const bool include_watch_only = GetLegacyScriptPubKeyMan() && IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + const isminetype is_mine_filter = include_watch_only ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; for (const COutPoint& output : lockedCoins) { auto it = mapWallet.find(output.hash); if (it != mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(locked_chain); + int depth = it->second.GetDepthInMainChain(); if (depth >= 0 && output.n < it->second.tx->vout.size() && - IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { + IsMine(it->second.tx->vout[output.n]) == is_mine_filter + ) { CTxDestination address; if (ExtractDestination(FindNonChangeParentOutput(*it->second.tx, output.n).scriptPubKey, address)) { result[address].emplace_back( @@ -2648,7 +2276,11 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil if (effective_value > 0) { group.fee += coin.m_input_bytes < 0 ? 0 : coin_selection_params.effective_fee.GetFee(coin.m_input_bytes); group.long_term_fee += coin.m_input_bytes < 0 ? 0 : long_term_feerate.GetFee(coin.m_input_bytes); - group.effective_value += effective_value; + if (coin_selection_params.m_subtract_fee_outputs) { + group.effective_value += coin.txout.nValue; + } else { + group.effective_value += effective_value; + } ++it; } else { it = group.Discard(coin); @@ -2674,13 +2306,14 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params, bool& bnb_used) const { std::vector<COutput> vCoins(vAvailableCoins); + CAmount value_to_select = nTargetValue; + + // Default to bnb was not used. If we use it, we set it later + bnb_used = false; // coin control -> return all selected outputs (we want all selected to go into the transaction for sure) if (coin_control.HasSelected() && !coin_control.fAllowOtherInputs) { - // We didn't use BnB here, so set it to false. - bnb_used = false; - for (const COutput& out : vCoins) { if (!out.fSpendable) @@ -2699,22 +2332,30 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm coin_control.ListSelected(vPresetInputs); for (const COutPoint& outpoint : vPresetInputs) { - // For now, don't use BnB if preset inputs are selected. TODO: Enable this later - bnb_used = false; - coin_selection_params.use_bnb = false; - std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); if (it != mapWallet.end()) { const CWalletTx& wtx = it->second; // Clearly invalid input, fail - if (wtx.tx->vout.size() <= outpoint.n) + if (wtx.tx->vout.size() <= outpoint.n) { return false; + } // Just to calculate the marginal byte size - nValueFromPresetInputs += wtx.tx->vout[outpoint.n].nValue; - setPresetCoins.insert(CInputCoin(wtx.tx, outpoint.n)); - } else + CInputCoin coin(wtx.tx, outpoint.n, wtx.GetSpendSize(outpoint.n, false)); + nValueFromPresetInputs += coin.txout.nValue; + if (coin.m_input_bytes <= 0) { + return false; // Not solvable, can't estimate size for fee + } + coin.effective_value = coin.txout.nValue - coin_selection_params.effective_fee.GetFee(coin.m_input_bytes); + if (coin_selection_params.use_bnb) { + value_to_select -= coin.effective_value; + } else { + value_to_select -= coin.txout.nValue; + } + setPresetCoins.insert(coin); + } else { return false; // TODO: Allow non-wallet inputs + } } // remove preset inputs from vCoins @@ -2743,14 +2384,14 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); - bool res = nTargetValue <= nValueFromPresetInputs || - SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(1, 1, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, 2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max()), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + bool res = value_to_select <= 0 || + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max()), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset util::insert(setCoinsRet, setPresetCoins); @@ -2775,7 +2416,14 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; SignatureData sigdata; - if (!ProduceSignature(*this, MutableTransactionSignatureCreator(&tx, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { + + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); + if (!provider) { + // We don't know about this scriptpbuKey; + return false; + } + + if (!ProduceSignature(*provider, MutableTransactionSignatureCreator(&tx, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { return false; } UpdateInput(input, sigdata); @@ -2927,7 +2575,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign) { CAmount nValue = 0; - ReserveDestination reservedest(this); + const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); + ReserveDestination reservedest(this, change_type); int nChangePosRequest = nChangePosInOut; unsigned int nSubtractFeeFromAmount = 0; for (const auto& recipient : vecSend) @@ -2986,8 +2635,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std return false; } CTxDestination dest; - const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); - bool ret = reservedest.GetReservedDestination(change_type, dest, true); + bool ret = reservedest.GetReservedDestination(dest, true); if (!ret) { strFailReason = "Keypool ran out, please call keypoolrefill first"; @@ -3010,7 +2658,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // BnB selector is the only selector used when this is true. // That should only happen on the first pass through the loop. - coin_selection_params.use_bnb = nSubtractFeeFromAmount == 0; // If we are doing subtract fee from recipient, then don't use BnB + coin_selection_params.use_bnb = true; + coin_selection_params.m_subtract_fee_outputs = nSubtractFeeFromAmount != 0; // If we are doing subtract fee from recipient, don't use effective values // Start with no fee and loop until there is enough fee while (true) { @@ -3024,7 +2673,9 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std nValueToSelect += nFeeRet; // vouts to the payees - coin_selection_params.tx_noinputs_size = 11; // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 output count, 1 witness overhead (dummy, flag, stack size) + if (!coin_selection_params.m_subtract_fee_outputs) { + coin_selection_params.tx_noinputs_size = 11; // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 output count, 1 witness overhead (dummy, flag, stack size) + } for (const auto& recipient : vecSend) { CTxOut txout(recipient.nAmount, recipient.scriptPubKey); @@ -3041,7 +2692,9 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } } // Include the fee cost for outputs. Note this is only used for BnB right now - coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); + if (!coin_selection_params.m_subtract_fee_outputs) { + coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); + } if (IsDust(txout, chain().relayDustFee())) { @@ -3060,7 +2713,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } // Choose coins to use - bool bnb_used; + bool bnb_used = false; if (pick_new_inputs) { nValueIn = 0; setCoins.clear(); @@ -3233,7 +2886,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(*this, MutableTransactionSignatureCreator(&txNew, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); + if (!provider || !ProduceSignature(*provider, MutableTransactionSignatureCreator(&txNew, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed").translated; return false; @@ -3318,7 +2972,7 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve } std::string err_string; - if (!wtx.SubmitMemoryPoolAndRelay(err_string, true, *locked_chain)) { + if (!wtx.SubmitMemoryPoolAndRelay(err_string, true)) { WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } @@ -3328,7 +2982,7 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { // Even if we don't use this lock in this function, we want to preserve // lock order in LoadToWallet if query of chain state is needed to know - // tx status. If lock can't be taken (e.g wallet-tool), tx confirmation + // tx status. If lock can't be taken (e.g bitcoin-wallet), tx confirmation // status may be not reliable. auto locked_chain = LockChain(); LOCK(cs_wallet); @@ -3339,20 +2993,17 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { if (database->Rewrite("\x04pool")) { - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); + } } } - { - LOCK(cs_KeyStore); - // This wallet is in its first run if all of these are empty - fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() - && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); + // This wallet is in its first run if there are no ScriptPubKeyMans and it isn't blank or no privkeys + fFirstRunRet = m_spk_managers.empty() && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); + if (fFirstRunRet) { + assert(m_external_spk_managers.empty()); + assert(m_internal_spk_managers.empty()); } if (nLoadWalletRet != DBErrors::LOAD_OK) @@ -3376,12 +3027,9 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { if (database->Rewrite("\x04pool")) { - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); + } } } @@ -3400,13 +3048,9 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { if (database->Rewrite("\x04pool")) { - LOCK(cs_wallet); - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); + } } } @@ -3427,7 +3071,7 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add if (!strPurpose.empty()) /* update purpose only if requested */ mapAddressBook[address].purpose = strPurpose; } - NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, + NotifyAddressBookChanged(this, address, strName, IsMine(address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose)) return false; @@ -3454,261 +3098,69 @@ bool CWallet::DelAddressBook(const CTxDestination& address) mapAddressBook.erase(address); } - NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); + NotifyAddressBookChanged(this, address, "", IsMine(address) != ISMINE_NO, "", CT_DELETED); WalletBatch(*database).ErasePurpose(EncodeDestination(address)); return WalletBatch(*database).EraseName(EncodeDestination(address)); } -/** - * Mark old keypool keys as used, - * and generate all new keys - */ -bool CWallet::NewKeyPool() -{ - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - return false; - } - { - LOCK(cs_wallet); - WalletBatch batch(*database); - - for (const int64_t nIndex : setInternalKeyPool) { - batch.ErasePool(nIndex); - } - setInternalKeyPool.clear(); - - for (const int64_t nIndex : setExternalKeyPool) { - batch.ErasePool(nIndex); - } - setExternalKeyPool.clear(); - - for (const int64_t nIndex : set_pre_split_keypool) { - batch.ErasePool(nIndex); - } - set_pre_split_keypool.clear(); - - m_pool_key_to_index.clear(); - - if (!TopUpKeyPool()) { - return false; - } - WalletLogPrintf("CWallet::NewKeyPool rewrote keypool\n"); - } - return true; -} - size_t CWallet::KeypoolCountExternalKeys() { AssertLockHeld(cs_wallet); - return setExternalKeyPool.size() + set_pre_split_keypool.size(); -} -void CWallet::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) -{ - AssertLockHeld(cs_wallet); - if (keypool.m_pre_split) { - set_pre_split_keypool.insert(nIndex); - } else if (keypool.fInternal) { - setInternalKeyPool.insert(nIndex); - } else { - setExternalKeyPool.insert(nIndex); + unsigned int count = 0; + for (auto spk_man : GetActiveScriptPubKeyMans()) { + count += spk_man->KeypoolCountExternalKeys(); } - m_max_keypool_index = std::max(m_max_keypool_index, nIndex); - m_pool_key_to_index[keypool.vchPubKey.GetID()] = nIndex; - // If no metadata exists yet, create a default with the pool key's - // creation time. Note that this may be overwritten by actually - // stored metadata for that key later, which is fine. - CKeyID keyid = keypool.vchPubKey.GetID(); - if (mapKeyMetadata.count(keyid) == 0) - mapKeyMetadata[keyid] = CKeyMetadata(keypool.nTime); + return count; } -bool CWallet::TopUpKeyPool(unsigned int kpSize) +unsigned int CWallet::GetKeyPoolSize() const { - if (!CanGenerateKeys()) { - return false; - } - { - LOCK(cs_wallet); - - if (IsLocked()) return false; - - // Top up key pool - unsigned int nTargetSize; - if (kpSize > 0) - nTargetSize = kpSize; - else - nTargetSize = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); - - // count amount of available keys (internal, external) - // make sure the keypool of external and internal keys fits the user selected target (-keypool) - int64_t missingExternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setExternalKeyPool.size(), (int64_t) 0); - int64_t missingInternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setInternalKeyPool.size(), (int64_t) 0); - - if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) - { - // don't create extra internal keys - missingInternal = 0; - } - bool internal = false; - WalletBatch batch(*database); - for (int64_t i = missingInternal + missingExternal; i--;) - { - if (i < missingInternal) { - internal = true; - } + AssertLockHeld(cs_wallet); - CPubKey pubkey(GenerateNewKey(batch, internal)); - AddKeypoolPubkeyWithDB(pubkey, internal, batch); - } - if (missingInternal + missingExternal > 0) { - WalletLogPrintf("keypool added %d keys (%d internal), size=%u (%u internal)\n", missingInternal + missingExternal, missingInternal, setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(), setInternalKeyPool.size()); - } + unsigned int count = 0; + for (auto spk_man : GetActiveScriptPubKeyMans()) { + count += spk_man->GetKeyPoolSize(); } - NotifyCanGetAddressesChanged(); - return true; + return count; } -void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) +bool CWallet::TopUpKeyPool(unsigned int kpSize) { LOCK(cs_wallet); - assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? - int64_t index = ++m_max_keypool_index; - if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { - throw std::runtime_error(std::string(__func__) + ": writing imported pubkey failed"); - } - if (internal) { - setInternalKeyPool.insert(index); - } else { - setExternalKeyPool.insert(index); - } - m_pool_key_to_index[pubkey.GetID()] = index; -} - -bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal) -{ - nIndex = -1; - keypool.vchPubKey = CPubKey(); - { - LOCK(cs_wallet); - - TopUpKeyPool(); - - bool fReturningInternal = fRequestedInternal; - fReturningInternal &= (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) || IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - bool use_split_keypool = set_pre_split_keypool.empty(); - std::set<int64_t>& setKeyPool = use_split_keypool ? (fReturningInternal ? setInternalKeyPool : setExternalKeyPool) : set_pre_split_keypool; - - // Get the oldest key - if (setKeyPool.empty()) { - return false; - } - - WalletBatch batch(*database); - - auto it = setKeyPool.begin(); - nIndex = *it; - setKeyPool.erase(it); - if (!batch.ReadPool(nIndex, keypool)) { - throw std::runtime_error(std::string(__func__) + ": read failed"); - } - CPubKey pk; - if (!GetPubKey(keypool.vchPubKey.GetID(), pk)) { - throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); - } - // If the key was pre-split keypool, we don't care about what type it is - if (use_split_keypool && keypool.fInternal != fReturningInternal) { - throw std::runtime_error(std::string(__func__) + ": keypool entry misclassified"); - } - if (!keypool.vchPubKey.IsValid()) { - throw std::runtime_error(std::string(__func__) + ": keypool entry invalid"); - } - - m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); - WalletLogPrintf("keypool reserve %d\n", nIndex); - } - NotifyCanGetAddressesChanged(); - return true; -} - -void CWallet::KeepKey(int64_t nIndex) -{ - // Remove from key pool - WalletBatch batch(*database); - batch.ErasePool(nIndex); - WalletLogPrintf("keypool keep %d\n", nIndex); -} - -void CWallet::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey) -{ - // Return to key pool - { - LOCK(cs_wallet); - if (fInternal) { - setInternalKeyPool.insert(nIndex); - } else if (!set_pre_split_keypool.empty()) { - set_pre_split_keypool.insert(nIndex); - } else { - setExternalKeyPool.insert(nIndex); - } - m_pool_key_to_index[pubkey.GetID()] = nIndex; - NotifyCanGetAddressesChanged(); + bool res = true; + for (auto spk_man : GetActiveScriptPubKeyMans()) { + res &= spk_man->TopUp(kpSize); } - WalletLogPrintf("keypool return %d\n", nIndex); -} - -bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) -{ - if (!CanGetAddresses(internal)) { - return false; - } - - CKeyPool keypool; - { - LOCK(cs_wallet); - int64_t nIndex; - if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - if (IsLocked()) return false; - WalletBatch batch(*database); - result = GenerateNewKey(batch, internal); - return true; - } - KeepKey(nIndex); - result = keypool.vchPubKey; - } - return true; + return res; } bool CWallet::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error) { LOCK(cs_wallet); error.clear(); - - TopUpKeyPool(); - - // Generate a new key that is added to wallet - CPubKey new_key; - if (!GetKeyFromPool(new_key)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; - return false; + bool result = false; + auto spk_man = GetScriptPubKeyMan(type, false /* internal */); + if (spk_man) { + spk_man->TopUp(); + result = spk_man->GetNewDestination(type, dest, error); + } + if (result) { + SetAddressBook(dest, label, "receive"); } - LearnRelatedScripts(new_key, type); - dest = GetDestinationForKey(new_key, type); - SetAddressBook(dest, label, "receive"); - return true; + return result; } bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error) { + LOCK(cs_wallet); error.clear(); - TopUpKeyPool(); - - ReserveDestination reservedest(this); - if (!reservedest.GetReservedDestination(type, dest, true)) { + ReserveDestination reservedest(this, type); + if (!reservedest.GetReservedDestination(dest, true)) { error = "Error: Keypool ran out, please call keypoolrefill first"; return false; } @@ -3717,36 +3169,28 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des return true; } -static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, WalletBatch& batch) { - if (setKeyPool.empty()) { - return GetTime(); - } - - CKeyPool keypool; - int64_t nIndex = *(setKeyPool.begin()); - if (!batch.ReadPool(nIndex, keypool)) { - throw std::runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); - } - assert(keypool.vchPubKey.IsValid()); - return keypool.nTime; -} - int64_t CWallet::GetOldestKeyPoolTime() { LOCK(cs_wallet); + int64_t oldestKey = std::numeric_limits<int64_t>::max(); + for (const auto& spk_man_pair : m_spk_managers) { + oldestKey = std::min(oldestKey, spk_man_pair.second->GetOldestKeyPoolTime()); + } + return oldestKey; +} - WalletBatch batch(*database); - - // load oldest key from keypool, get time and return - int64_t oldestKey = GetOldestKeyTimeInPool(setExternalKeyPool, batch); - if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) { - oldestKey = std::max(GetOldestKeyTimeInPool(setInternalKeyPool, batch), oldestKey); - if (!set_pre_split_keypool.empty()) { - oldestKey = std::max(GetOldestKeyTimeInPool(set_pre_split_keypool, batch), oldestKey); +void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations) { + for (auto& entry : mapWallet) { + CWalletTx& wtx = entry.second; + if (wtx.m_is_cache_empty) continue; + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { + CTxDestination dst; + if (ExtractDestination(wtx.tx->vout[i].scriptPubKey, dst) && destinations.count(dst)) { + wtx.MarkDirty(); + break; + } } } - - return oldestKey; } std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain::Lock& locked_chain) @@ -3755,17 +3199,18 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: { LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& walletEntry : mapWallet) { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain, trusted_parents)) continue; - if (wtx.IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase()) continue; - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; @@ -3777,7 +3222,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; + CAmount n = IsSpent(walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -3896,71 +3341,46 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co return result; } -bool ReserveDestination::GetReservedDestination(const OutputType type, CTxDestination& dest, bool internal) +bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal) { - if (!pwallet->CanGetAddresses(internal)) { + m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); + if (!m_spk_man) { return false; } + if (nIndex == -1) { + m_spk_man->TopUp(); + CKeyPool keypool; - if (!pwallet->ReserveKeyFromKeyPool(nIndex, keypool, internal)) { + if (!m_spk_man->GetReservedDestination(type, internal, address, nIndex, keypool)) { return false; } - vchPubKey = keypool.vchPubKey; fInternal = keypool.fInternal; } - assert(vchPubKey.IsValid()); - pwallet->LearnRelatedScripts(vchPubKey, type); - address = GetDestinationForKey(vchPubKey, type); dest = address; return true; } void ReserveDestination::KeepDestination() { - if (nIndex != -1) - pwallet->KeepKey(nIndex); + if (nIndex != -1) { + m_spk_man->KeepDestination(nIndex, type); + } nIndex = -1; - vchPubKey = CPubKey(); address = CNoDestination(); } void ReserveDestination::ReturnDestination() { if (nIndex != -1) { - pwallet->ReturnKey(nIndex, fInternal, vchPubKey); + m_spk_man->ReturnDestination(nIndex, fInternal, address); } nIndex = -1; - vchPubKey = CPubKey(); address = CNoDestination(); } -void CWallet::MarkReserveKeysAsUsed(int64_t keypool_id) -{ - AssertLockHeld(cs_wallet); - bool internal = setInternalKeyPool.count(keypool_id); - if (!internal) assert(setExternalKeyPool.count(keypool_id) || set_pre_split_keypool.count(keypool_id)); - std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); - auto it = setKeyPool->begin(); - - WalletBatch batch(*database); - while (it != std::end(*setKeyPool)) { - const int64_t& index = *(it); - if (index > keypool_id) break; // set*KeyPool is ordered - - CKeyPool keypool; - if (batch.ReadPool(index, keypool)) { //TODO: This should be unnecessary - m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); - } - LearnAllRelatedScripts(keypool.vchPubKey); - batch.ErasePool(index); - WalletLogPrintf("keypool index %d removed\n", index); - it = setKeyPool->erase(it); - } -} - void CWallet::LockCoin(const COutPoint& output) { AssertLockHeld(cs_wallet); @@ -4003,8 +3423,12 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C AssertLockHeld(cs_wallet); mapKeyBirth.clear(); + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + assert(spk_man != nullptr); + LOCK(spk_man->cs_KeyStore); + // get birth times for keys with metadata - for (const auto& entry : mapKeyMetadata) { + for (const auto& entry : spk_man->mapKeyMetadata) { if (entry.second.nCreateTime) { mapKeyBirth[entry.first] = entry.second.nCreateTime; } @@ -4014,7 +3438,7 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C const Optional<int> tip_height = locked_chain.getHeight(); const int max_height = tip_height && *tip_height > 144 ? *tip_height - 144 : 0; // the tip can be reorganized; use a 144-block safety margin std::map<CKeyID, int> mapKeyFirstBlock; - for (const CKeyID &keyid : GetKeys()) { + for (const CKeyID &keyid : spk_man->GetKeys()) { if (mapKeyBirth.count(keyid) == 0) mapKeyFirstBlock[keyid] = max_height; } @@ -4031,7 +3455,7 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C // ... which are already in a block for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs - for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *this)) { + for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { // ... and all their affected keys std::map<CKeyID, int>::iterator rit = mapKeyFirstBlock.find(keyid); if (rit != mapKeyFirstBlock.end() && *height < rit->second) @@ -4106,20 +3530,20 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const return nTimeSmart; } -bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, const std::string &value) +bool CWallet::AddDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key, const std::string &value) { if (boost::get<CNoDestination>(&dest)) return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - return WalletBatch(*database).WriteDestData(EncodeDestination(dest), key, value); + return batch.WriteDestData(EncodeDestination(dest), key, value); } -bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) +bool CWallet::EraseDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - return WalletBatch(*database).EraseDestData(EncodeDestination(dest), key); + return batch.EraseDestData(EncodeDestination(dest), key); } void CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -4156,24 +3580,6 @@ std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const return values; } -void CWallet::MarkPreSplitKeys() -{ - WalletBatch batch(*database); - for (auto it = setExternalKeyPool.begin(); it != setExternalKeyPool.end();) { - int64_t index = *it; - CKeyPool keypool; - if (!batch.ReadPool(index, keypool)) { - throw std::runtime_error(std::string(__func__) + ": read keypool entry failed"); - } - keypool.m_pre_split = true; - if (!batch.WritePool(index, keypool)) { - throw std::runtime_error(std::string(__func__) + ": writing modified keypool entry failed"); - } - set_pre_split_keypool.insert(index); - it = setExternalKeyPool.erase(it); - } -} - bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::vector<std::string>& warnings) { // Do some checking on wallet path. It should be either a: @@ -4314,31 +3720,8 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } - bool hd_upgrade = false; - bool split_upgrade = false; - if (walletInstance->CanSupportFeature(FEATURE_HD) && !walletInstance->IsHDEnabled()) { - walletInstance->WalletLogPrintf("Upgrading wallet to HD\n"); - walletInstance->SetMinVersion(FEATURE_HD); - - // generate a new master key - CPubKey masterPubKey = walletInstance->GenerateNewSeed(); - walletInstance->SetHDSeed(masterPubKey); - hd_upgrade = true; - } - // Upgrade to HD chain split if necessary - if (walletInstance->CanSupportFeature(FEATURE_HD_SPLIT)) { - walletInstance->WalletLogPrintf("Upgrading wallet to use HD chain split\n"); - walletInstance->SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); - split_upgrade = FEATURE_HD_SPLIT > prev_version; - } - // Mark all keys currently in the keypool as pre-split - if (split_upgrade) { - walletInstance->MarkPreSplitKeys(); - } - // Regenerate the keypool if upgraded to HD - if (hd_upgrade) { - if (!walletInstance->TopUpKeyPool()) { - error = _("Unable to generate keys").translated; + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (!spk_man->Upgrade(prev_version, error)) { return nullptr; } } @@ -4350,16 +3733,18 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetMinVersion(FEATURE_LATEST); walletInstance->SetWalletFlags(wallet_creation_flags, false); - if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { - // generate a new seed - CPubKey seed = walletInstance->GenerateNewSeed(); - walletInstance->SetHDSeed(seed); - } - // Top up the keypool - if (walletInstance->CanGenerateKeys() && !walletInstance->TopUpKeyPool()) { - error = _("Unable to generate initial keys").translated; - return nullptr; + // Always create LegacyScriptPubKeyMan for now + walletInstance->SetupLegacyScriptPubKeyMan(); + + if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { + LOCK(walletInstance->cs_wallet); + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = _("Unable to generate initial keys").translated; + return nullptr; + } + } } auto locked_chain = chain.lock(); @@ -4369,9 +3754,11 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - LOCK(walletInstance->cs_KeyStore); - if (!walletInstance->mapKeys.empty() || !walletInstance->mapCryptedKeys.empty()) { - warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (spk_man->HavePrivateKeys()) { + warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + break; + } } } @@ -4492,8 +3879,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const Optional<int> tip_height = locked_chain->getHeight(); if (tip_height) { walletInstance->m_last_block_processed = locked_chain->getBlockHash(*tip_height); + walletInstance->m_last_block_processed_height = *tip_height; } else { walletInstance->m_last_block_processed.SetNull(); + walletInstance->m_last_block_processed_height = -1; } if (tip_height && *tip_height != rescan_height) @@ -4521,8 +3910,13 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - if (walletInstance->nTimeFirstKey) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height, nullptr)) { + Optional<int64_t> time_first_key; + for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { + int64_t time = spk_man->GetTimeFirstKey(); + if (!time_first_key || time < *time_first_key) time_first_key = time; + } + if (time_first_key) { + if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } @@ -4562,7 +3956,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - chain.loadWallet(interfaces::MakeWallet(walletInstance)); + { + LOCK(cs_wallets); + for (auto& load_wallet : g_load_wallet_fns) { + load_wallet(interfaces::MakeWallet(walletInstance)); + } + } // Register with the validation interface. It's ok to do this after rescan since we're still holding locked_chain. walletInstance->handleNotifications(); @@ -4590,7 +3989,7 @@ void CWallet::postInitProcess() // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded - ReacceptWalletTransactions(*locked_chain); + ReacceptWalletTransactions(); // Update wallet transactions with current mempool transactions. chain().requestMempoolTransactions(*this); @@ -4616,55 +4015,28 @@ CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) m_pre_split = false; } -void CWalletTx::SetConf(Status status, const uint256& block_hash, int posInBlock) -{ - // Update tx status - m_confirm.status = status; - - // Update the tx's hashBlock - m_confirm.hashBlock = block_hash; - - // set the position of the transaction in the block - m_confirm.nIndex = posInBlock; -} - -int CWalletTx::GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const +int CWalletTx::GetDepthInMainChain() const { + assert(pwallet != nullptr); + AssertLockHeld(pwallet->cs_wallet); if (isUnconfirmed() || isAbandoned()) return 0; - return locked_chain.getBlockDepth(m_confirm.hashBlock) * (isConflicted() ? -1 : 1); + return (pwallet->GetLastBlockHeight() - m_confirm.block_height + 1) * (isConflicted() ? -1 : 1); } -int CWalletTx::GetBlocksToMaturity(interfaces::Chain::Lock& locked_chain) const +int CWalletTx::GetBlocksToMaturity() const { if (!IsCoinBase()) return 0; - int chain_depth = GetDepthInMainChain(locked_chain); + int chain_depth = GetDepthInMainChain(); assert(chain_depth >= 0); // coinbase tx should not be conflicted return std::max(0, (COINBASE_MATURITY+1) - chain_depth); } -bool CWalletTx::IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const +bool CWalletTx::IsImmatureCoinBase() const { // note GetBlocksToMaturity is 0 for non-coinbase tx - return GetBlocksToMaturity(locked_chain) > 0; -} - -void CWallet::LearnRelatedScripts(const CPubKey& key, OutputType type) -{ - if (key.IsCompressed() && (type == OutputType::P2SH_SEGWIT || type == OutputType::BECH32)) { - CTxDestination witdest = WitnessV0KeyHash(key.GetID()); - CScript witprog = GetScriptForDestination(witdest); - // Make sure the resulting program is solvable. - assert(IsSolvable(*this, witprog)); - AddCScript(witprog); - } -} - -void CWallet::LearnAllRelatedScripts(const CPubKey& key) -{ - // OutputType::P2SH_SEGWIT always adds all necessary scripts for all types. - LearnRelatedScripts(key, OutputType::P2SH_SEGWIT); + return GetBlocksToMaturity() > 0; } std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const { @@ -4697,44 +4069,9 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu return groups; } -bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const -{ - CKeyMetadata meta; - { - LOCK(cs_wallet); - auto it = mapKeyMetadata.find(keyID); - if (it != mapKeyMetadata.end()) { - meta = it->second; - } - } - if (meta.has_key_origin) { - std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); - info.path = meta.key_origin.path; - } else { // Single pubkeys get the master fingerprint of themselves - std::copy(keyID.begin(), keyID.begin() + 4, info.fingerprint); - } - return true; -} - -bool CWallet::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) +bool CWallet::IsCrypted() const { - LOCK(cs_wallet); - std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); - mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; - mapKeyMetadata[pubkey.GetID()].has_key_origin = true; - mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); - return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); -} - -bool CWallet::SetCrypted() -{ - LOCK(cs_KeyStore); - if (fUseCrypto) - return true; - if (!mapKeys.empty()) - return false; - fUseCrypto = true; - return true; + return HasEncryptionKeys(); } bool CWallet::IsLocked() const @@ -4742,17 +4079,17 @@ bool CWallet::IsLocked() const if (!IsCrypted()) { return false; } - LOCK(cs_KeyStore); + LOCK(cs_wallet); return vMasterKey.empty(); } bool CWallet::Lock() { - if (!SetCrypted()) + if (!IsCrypted()) return false; { - LOCK(cs_KeyStore); + LOCK(cs_wallet); vMasterKey.clear(); } @@ -4763,165 +4100,130 @@ bool CWallet::Lock() bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) { { - LOCK(cs_KeyStore); - if (!SetCrypted()) - return false; - - bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys - bool keyFail = false; - CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); - for (; mi != mapCryptedKeys.end(); ++mi) - { - const CPubKey &vchPubKey = (*mi).second.first; - const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; - CKey key; - if (!DecryptKey(vMasterKeyIn, vchCryptedSecret, vchPubKey, key)) - { - keyFail = true; - break; + LOCK(cs_wallet); + for (const auto& spk_man_pair : m_spk_managers) { + if (!spk_man_pair.second->CheckDecryptionKey(vMasterKeyIn, accept_no_keys)) { + return false; } - keyPass = true; - if (fDecryptionThoroughlyChecked) - break; } - if (keyPass && keyFail) - { - LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); - throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); - } - if (keyFail || (!keyPass && !accept_no_keys)) - return false; vMasterKey = vMasterKeyIn; - fDecryptionThoroughlyChecked = true; } NotifyStatusChanged(this); return true; } -bool CWallet::HaveKey(const CKeyID &address) const +std::set<ScriptPubKeyMan*> CWallet::GetActiveScriptPubKeyMans() const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::HaveKey(address); + std::set<ScriptPubKeyMan*> spk_mans; + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_man = GetScriptPubKeyMan(t, internal); + if (spk_man) { + spk_mans.insert(spk_man); + } + } } - return mapCryptedKeys.count(address) > 0; + return spk_mans; } -bool CWallet::GetKey(const CKeyID &address, CKey& keyOut) const +std::set<ScriptPubKeyMan*> CWallet::GetAllScriptPubKeyMans() const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::GetKey(address, keyOut); + std::set<ScriptPubKeyMan*> spk_mans; + for (const auto& spk_man_pair : m_spk_managers) { + spk_mans.insert(spk_man_pair.second.get()); } - - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - const CPubKey &vchPubKey = (*mi).second.first; - const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; - return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); - } - return false; + return spk_mans; } -bool CWallet::GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const OutputType& type, bool internal) const { - LOCK(cs_KeyStore); - WatchKeyMap::const_iterator it = mapWatchKeys.find(address); - if (it != mapWatchKeys.end()) { - pubkey_out = it->second; - return true; + const std::map<OutputType, ScriptPubKeyMan*>& spk_managers = internal ? m_internal_spk_managers : m_external_spk_managers; + std::map<OutputType, ScriptPubKeyMan*>::const_iterator it = spk_managers.find(type); + if (it == spk_managers.end()) { + WalletLogPrintf("%s scriptPubKey Manager for output type %d does not exist\n", internal ? "Internal" : "External", static_cast<int>(type)); + return nullptr; } - return false; + return it->second; } -bool CWallet::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const CScript& script) const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - if (!FillableSigningProvider::GetPubKey(address, vchPubKeyOut)) { - return GetWatchPubKey(address, vchPubKeyOut); + SignatureData sigdata; + for (const auto& spk_man_pair : m_spk_managers) { + if (spk_man_pair.second->CanProvide(script, sigdata)) { + return spk_man_pair.second.get(); } - return true; - } - - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - vchPubKeyOut = (*mi).second.first; - return true; } - // Check for watch-only pubkeys - return GetWatchPubKey(address, vchPubKeyOut); + return nullptr; } -std::set<CKeyID> CWallet::GetKeys() const +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const uint256& id) const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::GetKeys(); + if (m_spk_managers.count(id) > 0) { + return m_spk_managers.at(id).get(); } - std::set<CKeyID> set_address; - for (const auto& mi : mapCryptedKeys) { - set_address.insert(mi.first); - } - return set_address; + return nullptr; } -bool CWallet::EncryptKeys(CKeyingMaterial& vMasterKeyIn) +std::unique_ptr<SigningProvider> CWallet::GetSigningProvider(const CScript& script) const { - LOCK(cs_KeyStore); - if (!mapCryptedKeys.empty() || IsCrypted()) - return false; + SignatureData sigdata; + return GetSigningProvider(script, sigdata); +} - fUseCrypto = true; - for (const KeyMap::value_type& mKey : mapKeys) - { - const CKey &key = mKey.second; - CPubKey vchPubKey = key.GetPubKey(); - CKeyingMaterial vchSecret(key.begin(), key.end()); - std::vector<unsigned char> vchCryptedSecret; - if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) - return false; - if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) - return false; +std::unique_ptr<SigningProvider> CWallet::GetSigningProvider(const CScript& script, SignatureData& sigdata) const +{ + for (const auto& spk_man_pair : m_spk_managers) { + if (spk_man_pair.second->CanProvide(script, sigdata)) { + return spk_man_pair.second->GetSigningProvider(script); + } } - mapKeys.clear(); - return true; + return nullptr; } -bool CWallet::AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey) +LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::AddKeyPubKey(key, pubkey); - } + // Legacy wallets only have one ScriptPubKeyMan which is a LegacyScriptPubKeyMan. + // Everything in m_internal_spk_managers and m_external_spk_managers point to the same legacyScriptPubKeyMan. + auto it = m_internal_spk_managers.find(OutputType::LEGACY); + if (it == m_internal_spk_managers.end()) return nullptr; + return dynamic_cast<LegacyScriptPubKeyMan*>(it->second); +} - if (IsLocked()) { - return false; - } +LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() +{ + SetupLegacyScriptPubKeyMan(); + return GetLegacyScriptPubKeyMan(); +} - std::vector<unsigned char> vchCryptedSecret; - CKeyingMaterial vchSecret(key.begin(), key.end()); - if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) { - return false; +void CWallet::SetupLegacyScriptPubKeyMan() +{ + if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty()) { + return; } - if (!AddCryptedKey(pubkey, vchCryptedSecret)) { - return false; + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new LegacyScriptPubKeyMan(*this)); + for (const auto& type : OUTPUT_TYPES) { + m_internal_spk_managers[type] = spk_manager.get(); + m_external_spk_managers[type] = spk_manager.get(); } - return true; + m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); } +const CKeyingMaterial& CWallet::GetEncryptionKey() const +{ + return vMasterKey; +} -bool CWallet::AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +bool CWallet::HasEncryptionKeys() const { - LOCK(cs_KeyStore); - if (!SetCrypted()) { - return false; - } + return !mapMasterKeys.empty(); +} - mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); - ImplicitlyLearnRelatedKeyScripts(vchPubKey); - return true; +void CWallet::ConnectScriptPubKeyManNotifiers() +{ + for (const auto& spk_man : GetActiveScriptPubKeyMans()) { + spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); + spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); + } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 85c277ff50..a918bb8833 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ #include <validationinterface.h> #include <wallet/coinselection.h> #include <wallet/crypter.h> -#include <wallet/ismine.h> +#include <wallet/scriptpubkeyman.h> #include <wallet/walletdb.h> #include <wallet/walletutil.h> @@ -35,6 +35,8 @@ #include <boost/signals2/signal.hpp> +using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; + //! Explicitly unload and delete the wallet. //! Blocks the current thread after signaling the unload intent so that all //! wallet clients release the wallet. @@ -48,6 +50,7 @@ bool HasWallets(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings); +std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet); enum class WalletCreationStatus { SUCCESS, @@ -57,8 +60,6 @@ enum class WalletCreationStatus { WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); -//! Default for -keypool -static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; //! -paytxfee default constexpr CAmount DEFAULT_PAY_TX_FEE = 0; //! -fallbackfee default @@ -73,8 +74,6 @@ static const CAmount WALLET_INCREMENTAL_RELAY_FEE = 5000; static const bool DEFAULT_SPEND_ZEROCONF_CHANGE = true; //! Default for -walletrejectlongchains static const bool DEFAULT_WALLET_REJECT_LONG_CHAINS = false; -//! Default for -avoidpartialspends -static const bool DEFAULT_AVOIDPARTIALSPENDS = false; //! -txconfirmtarget default static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; //! -walletrbf default @@ -99,58 +98,12 @@ struct FeeCalculation; enum class FeeEstimateMode; class ReserveDestination; -/** (client) version numbers for particular wallet features */ -enum WalletFeature -{ - FEATURE_BASE = 10500, // the earliest version new wallets supports (only useful for getwalletinfo's clientversion output) - - FEATURE_WALLETCRYPT = 40000, // wallet encryption - FEATURE_COMPRPUBKEY = 60000, // compressed public keys - - FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) - - FEATURE_HD_SPLIT = 139900, // Wallet with HD chain split (change outputs will use m/0'/1'/k) - - FEATURE_NO_DEFAULT_KEY = 159900, // Wallet without a default key written - - FEATURE_PRE_SPLIT_KEYPOOL = 169900, // Upgraded to HD SPLIT and can have a pre-split keypool - - FEATURE_LATEST = FEATURE_PRE_SPLIT_KEYPOOL -}; - //! Default for -addresstype constexpr OutputType DEFAULT_ADDRESS_TYPE{OutputType::BECH32}; //! Default for -changetype constexpr OutputType DEFAULT_CHANGE_TYPE{OutputType::CHANGE_AUTO}; -enum WalletFlags : uint64_t { - // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown - // unknown wallet flags in the lower section <= (1 << 31) will be tolerated - - // will categorize coins as clean (not reused) and dirty (reused), and handle - // them with privacy considerations in mind - WALLET_FLAG_AVOID_REUSE = (1ULL << 0), - - // Indicates that the metadata has already been upgraded to contain key origins - WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), - - // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) - WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), - - //! Flag set when a wallet contains no HD seed and no private keys, scripts, - //! addresses, and other watch only things, and is therefore "blank." - //! - //! The only function this flag serves is to distinguish a blank wallet from - //! a newly created wallet when the wallet database is loaded, to avoid - //! initialization that should only happen on first run. - //! - //! This flag is also a mandatory flag to prevent previous versions of - //! bitcoin from opening the wallet, thinking it was newly created, and - //! then improperly reinitializing it. - WALLET_FLAG_BLANK_WALLET = (1ULL << 33), -}; - static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET @@ -169,99 +122,6 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ extern const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS; -/** A key from a CWallet's keypool - * - * The wallet holds one (for pre HD-split wallets) or several keypools. These - * are sets of keys that have not yet been used to provide addresses or receive - * change. - * - * The Bitcoin Core wallet was originally a collection of unrelated private - * keys with their associated addresses. If a non-HD wallet generated a - * key/address, gave that address out and then restored a backup from before - * that key's generation, then any funds sent to that address would be - * lost definitively. - * - * The keypool was implemented to avoid this scenario (commit: 10384941). The - * wallet would generate a set of keys (100 by default). When a new public key - * was required, either to give out as an address or to use in a change output, - * it would be drawn from the keypool. The keypool would then be topped up to - * maintain 100 keys. This ensured that as long as the wallet hadn't used more - * than 100 keys since the previous backup, all funds would be safe, since a - * restored wallet would be able to scan for all owned addresses. - * - * A keypool also allowed encrypted wallets to give out addresses without - * having to be decrypted to generate a new private key. - * - * With the introduction of HD wallets (commit: f1902510), the keypool - * essentially became an address look-ahead pool. Restoring old backups can no - * longer definitively lose funds as long as the addresses used were from the - * wallet's HD seed (since all private keys can be rederived from the seed). - * However, if many addresses were used since the backup, then the wallet may - * not know how far ahead in the HD chain to look for its addresses. The - * keypool is used to implement a 'gap limit'. The keypool maintains a set of - * keys (by default 1000) ahead of the last used key and scans for the - * addresses of those keys. This avoids the risk of not seeing transactions - * involving the wallet's addresses, or of re-using the same address. - * - * The HD-split wallet feature added a second keypool (commit: 02592f4c). There - * is an external keypool (for addresses to hand out) and an internal keypool - * (for change addresses). - * - * Keypool keys are stored in the wallet/keystore's keymap. The keypool data is - * stored as sets of indexes in the wallet (setInternalKeyPool, - * setExternalKeyPool and set_pre_split_keypool), and a map from the key to the - * index (m_pool_key_to_index). The CKeyPool object is used to - * serialize/deserialize the pool data to/from the database. - */ -class CKeyPool -{ -public: - //! The time at which the key was generated. Set in AddKeypoolPubKeyWithDB - int64_t nTime; - //! The public key - CPubKey vchPubKey; - //! Whether this keypool entry is in the internal keypool (for change outputs) - bool fInternal; - //! Whether this key was generated for a keypool before the wallet was upgraded to HD-split - bool m_pre_split; - - CKeyPool(); - CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(fInternal); - } - catch (std::ios_base::failure&) { - /* flag as external address if we can't read the internal boolean - (this will be the case for any wallet before the HD chain split version) */ - fInternal = false; - } - try { - READWRITE(m_pre_split); - } - catch (std::ios_base::failure&) { - /* flag as postsplit address if we can't read the m_pre_split boolean - (this will be the case for any wallet that upgrades to HD chain split)*/ - m_pre_split = false; - } - } - else { - READWRITE(fInternal); - READWRITE(m_pre_split); - } - } -}; - /** A wrapper to reserve an address from a wallet * * ReserveDestination is used to reserve an address. @@ -281,11 +141,12 @@ class ReserveDestination { protected: //! The wallet to reserve from - CWallet* pwallet; + CWallet* const pwallet; + //! The ScriptPubKeyMan to reserve from. Based on type when GetReservedDestination is called + ScriptPubKeyMan* m_spk_man{nullptr}; + OutputType const type; //! The index of the address's key in the keypool int64_t nIndex{-1}; - //! The public key for the address - CPubKey vchPubKey; //! The destination CTxDestination address; //! Whether this is from the internal (change output) keypool @@ -293,10 +154,9 @@ protected: public: //! Construct a ReserveDestination object. This does NOT reserve an address yet - explicit ReserveDestination(CWallet* pwalletIn) - { - pwallet = pwalletIn; - } + explicit ReserveDestination(CWallet* pwallet, OutputType type) + : pwallet(pwallet) + , type(type) { } ReserveDestination(const ReserveDestination&) = delete; ReserveDestination& operator=(const ReserveDestination&) = delete; @@ -308,7 +168,7 @@ public: } //! Reserve an address - bool GetReservedDestination(const OutputType type, CTxDestination& pubkey, bool internal); + bool GetReservedDestination(CTxDestination& pubkey, bool internal); //! Return reserved address void ReturnDestination(); //! Keep the address. Do not return it's key to the keypool when this object goes out of scope @@ -453,6 +313,13 @@ public: enum AmountType { DEBIT, CREDIT, IMMATURE_CREDIT, AVAILABLE_CREDIT, AMOUNTTYPE_ENUM_ELEMENTS }; CAmount GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate = false) const; mutable CachableAmount m_amounts[AMOUNTTYPE_ENUM_ELEMENTS]; + /** + * This flag is true if all m_amounts caches are empty. This is particularly + * useful in places where MarkDirty is conditionally called and the + * condition can be expensive and thus can be skipped if the flag is true. + * See MarkDestinationsDirty. + */ + mutable bool m_is_cache_empty{true}; mutable bool fChangeCached; mutable bool fInMempool; mutable CAmount nChangeCached; @@ -496,14 +363,17 @@ public: ABANDONED }; - /* Confirmation includes tx status and a pair of {block hash/tx index in block} at which tx has been confirmed. - * This pair is both 0 if tx hasn't confirmed yet. Meaning of these fields changes with CONFLICTED state - * where they instead point to block hash and index of the deepest conflicting tx. + /* Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} + * at which tx has been confirmed. All three are set to 0 if tx is unconfirmed or abandoned. + * Meaning of these fields changes with CONFLICTED state where they instead point to block hash + * and block height of the deepest conflicting tx. */ struct Confirmation { - Status status = UNCONFIRMED; - uint256 hashBlock = uint256(); - int nIndex = 0; + Status status; + int block_height; + uint256 hashBlock; + int nIndex; + Confirmation(Status s = UNCONFIRMED, int b = 0, uint256 h = uint256(), int i = 0) : status(s), block_height(b), hashBlock(h), nIndex(i) {} }; Confirmation m_confirm; @@ -546,7 +416,6 @@ public: * compatibility (pre-commit 9ac63d6). */ if (serializedIndex == -1 && m_confirm.hashBlock == ABANDON_HASH) { - m_confirm.hashBlock = uint256(); setAbandoned(); } else if (serializedIndex == -1) { setConflicted(); @@ -577,6 +446,7 @@ public: m_amounts[IMMATURE_CREDIT].Reset(); m_amounts[AVAILABLE_CREDIT].Reset(); fChangeCached = false; + m_is_cache_empty = true; } void BindWallet(CWallet *pwalletIn) @@ -587,14 +457,14 @@ public: //! filter decides which addresses will count towards the debit CAmount GetDebit(const isminefilter& filter) const; - CAmount GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const; - CAmount GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true) const; + CAmount GetCredit(const isminefilter& filter) const; + CAmount GetImmatureCredit(bool fUseCache = true) const; // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid // having to resolve the issue of member access into incomplete type CWallet. - CAmount GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true, const isminefilter& filter=ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; - CAmount GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache=true) const; + CAmount GetAvailableCredit(bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; + CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const; CAmount GetChange() const; // Get the marginal bytes if spending the specified output from this transaction @@ -616,11 +486,12 @@ public: bool InMempool() const; bool IsTrusted(interfaces::Chain::Lock& locked_chain) const; + bool IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; // Pass this transaction to node for mempool insertion and relay to peers if flag set to true - bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, interfaces::Chain::Lock& locked_chain); + bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation @@ -630,38 +501,44 @@ public: // in place. std::set<uint256> GetConflicts() const NO_THREAD_SAFETY_ANALYSIS; - void SetConf(Status status, const uint256& block_hash, int posInBlock); - /** * Return depth of transaction in blockchain: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain */ - int GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const; - bool IsInMainChain(interfaces::Chain::Lock& locked_chain) const { return GetDepthInMainChain(locked_chain) > 0; } + // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation + // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to + // resolve the issue of member access into incomplete type CWallet. Note + // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" + // in place. + int GetDepthInMainChain() const NO_THREAD_SAFETY_ANALYSIS; + bool IsInMainChain() const { return GetDepthInMainChain() > 0; } /** * @return number of blocks to maturity for this transaction: * 0 : is not a coinbase transaction, or is a mature coinbase transaction * >0 : is a coinbase transaction which matures in this many blocks */ - int GetBlocksToMaturity(interfaces::Chain::Lock& locked_chain) const; + int GetBlocksToMaturity() const; bool isAbandoned() const { return m_confirm.status == CWalletTx::ABANDONED; } void setAbandoned() { m_confirm.status = CWalletTx::ABANDONED; m_confirm.hashBlock = uint256(); + m_confirm.block_height = 0; m_confirm.nIndex = 0; } bool isConflicted() const { return m_confirm.status == CWalletTx::CONFLICTED; } void setConflicted() { m_confirm.status = CWalletTx::CONFLICTED; } bool isUnconfirmed() const { return m_confirm.status == CWalletTx::UNCONFIRMED; } void setUnconfirmed() { m_confirm.status = CWalletTx::UNCONFIRMED; } + bool isConfirmed() const { return m_confirm.status == CWalletTx::CONFIRMED; } void setConfirmed() { m_confirm.status = CWalletTx::CONFIRMED; } const uint256& GetHash() const { return tx->GetHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } - bool IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const; + bool IsImmatureCoinBase() const; }; class COutput @@ -715,6 +592,8 @@ struct CoinSelectionParams size_t change_spend_size = 0; CFeeRate effective_fee = CFeeRate(0); size_t tx_noinputs_size = 0; + //! Indicate that we are subtracting the fee from outputs + bool m_subtract_fee_outputs = false; CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size) : use_bnb(use_bnb), change_output_size(change_output_size), change_spend_size(change_spend_size), effective_fee(effective_fee), tx_noinputs_size(tx_noinputs_size) {} CoinSelectionParams() {} @@ -722,37 +601,15 @@ struct CoinSelectionParams class WalletRescanReserver; //forward declarations for ScanForWalletTransactions/RescanFromTime /** - * A CWallet is an extension of a keystore, which also maintains a set of transactions and balances, - * and provides the ability to create new transactions. + * A CWallet maintains a set of transactions and balances, and provides the ability to create new transactions. */ -class CWallet final : public FillableSigningProvider, private interfaces::Chain::Notifications +class CWallet final : public WalletStorage, private interfaces::Chain::Notifications { private: - CKeyingMaterial vMasterKey GUARDED_BY(cs_KeyStore); - - //! if fUseCrypto is true, mapKeys must be empty - //! if fUseCrypto is false, vMasterKey must be empty - std::atomic<bool> fUseCrypto; - - //! keeps track of whether Unlock has run a thorough check before - bool fDecryptionThoroughlyChecked; - - using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; - using WatchOnlySet = std::set<CScript>; - using WatchKeyMap = std::map<CKeyID, CPubKey>; + CKeyingMaterial vMasterKey GUARDED_BY(cs_wallet); - bool SetCrypted(); - - //! will encrypt previously unencrypted keys - bool EncryptKeys(CKeyingMaterial& vMasterKeyIn); bool Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys = false); - CryptedKeyMap mapCryptedKeys GUARDED_BY(cs_KeyStore); - WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); - WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); - - bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); std::atomic<bool> fAbortRescan{false}; std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver @@ -761,8 +618,6 @@ private: std::mutex mutexScanning; friend class WalletRescanReserver; - WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; - //! the current wallet version: clients below this version are not able to load the wallet int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; @@ -798,10 +653,10 @@ private: * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. */ - bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /* Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ - void MarkConflicted(const uint256& hashBlock, const uint256& hashTx); + void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); /* Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -810,55 +665,18 @@ private: /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ - void SyncTransaction(const CTransactionRef& tx, CWalletTx::Status status, const uint256& block_hash, int posInBlock = 0, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /* the HD chain data model (external chain counters) */ - CHDChain hdChain; - - /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); - int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; - std::map<CKeyID, int64_t> m_pool_key_to_index; std::atomic<uint64_t> m_wallet_flags{0}; - int64_t nTimeFirstKey GUARDED_BY(cs_wallet) = 0; - - /** - * Private version of AddWatchOnly method which does not accept a - * timestamp, and which will reset the wallet's nTimeFirstKey value to 1 if - * the watch key did not previously have a timestamp associated with it. - * Because this is an inherited virtual method, it is accessible despite - * being marked private, but it is marked private anyway to encourage use - * of the other AddWatchOnly which accepts a timestamp and sets - * nTimeFirstKey more intelligently for more efficient rescans. - */ - bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnlyInMem(const CScript &dest); - - /** Add a KeyOriginInfo to the wallet */ - bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); - - //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); - bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose); - //! Adds a script to the store and saves it to disk - bool AddCScriptWithDB(WalletBatch& batch, const CScript& script); - //! Unsets a wallet flag and saves it to disk void UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag); + //! Unset the blank wallet flag and saves it to disk + void UnsetBlankWalletFlag(WalletBatch& batch) override; + /** Interface for accessing chain state. */ interfaces::Chain* m_chain; @@ -872,21 +690,31 @@ private: * The following is used to keep track of how far behind the wallet is * from the chain sync, and to allow clients to block on us being caught up. * - * Note that this is *not* how far we've processed, we may need some rescan - * to have seen all transactions in the chain, but is only used to track - * live BlockConnected callbacks. + * Processed hash is a pointer on node's tip and doesn't imply that the wallet + * has scanned sequentially all blocks up to this one. */ uint256 m_last_block_processed GUARDED_BY(cs_wallet); - //! Fetches a key from the keypool - bool GetKeyFromPool(CPubKey &key, bool internal = false); + /* Height of last block processed is used by wallet to know depth of transactions + * without relying on Chain interface beyond asynchronous updates. For safety, we + * initialize it to -1. Height is a pointer on node's tip and doesn't imply + * that the wallet has scanned sequentially all blocks up to this one. + */ + int m_last_block_processed_height GUARDED_BY(cs_wallet) = -1; + + std::map<OutputType, ScriptPubKeyMan*> m_external_spk_managers; + std::map<OutputType, ScriptPubKeyMan*> m_internal_spk_managers; + + // Indexed by a unique identifier produced by each ScriptPubKeyMan using + // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure + std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; public: /* * Main wallet lock. * This lock protects all the fields added by CWallet. */ - mutable CCriticalSection cs_wallet; + mutable RecursiveMutex cs_wallet; /** Get database handle used by this wallet. Ideally this function would * not be necessary. @@ -895,6 +723,7 @@ public: { return *database; } + WalletDatabase& GetDatabase() override { return *database; } /** * Select a set of coins such that nValueRet >= nTargetValue and at least @@ -910,24 +739,13 @@ public: */ const std::string& GetName() const { return m_location.GetName(); } - void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - // Map from Key ID to key metadata. - std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_wallet); - - // Map from Script ID to key metadata (for watch-only keys). - std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); - typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; /** Construct wallet with specified name and database implementation. */ CWallet(interfaces::Chain* chain, const WalletLocation& location, std::unique_ptr<WalletDatabase> database) - : fUseCrypto(false), - fDecryptionThoroughlyChecked(false), - m_chain(chain), + : m_chain(chain), m_location(location), database(std::move(database)) { @@ -937,12 +755,10 @@ public: { // Should not have slots connected at this point. assert(NotifyUnload.empty()); - delete encrypted_batch; - encrypted_batch = nullptr; } - bool IsCrypted() const { return fUseCrypto; } - bool IsLocked() const; + bool IsCrypted() const; + bool IsLocked() const override; bool Lock(); /** Interface to assert chain access and if successful lock it */ @@ -972,7 +788,7 @@ public: const CWalletTx* GetWalletTx(const uint256& hash) const; //! check whether we are allowed to upgrade (or already support) to the named feature - bool CanSupportFeature(enum WalletFeature wf) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } + bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } /** * populate vCoins with vector of available COutputs. @@ -998,12 +814,11 @@ public: bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; - bool IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - // Whether this or any UTXO with the same CTxDestination has been spent. - bool IsUsedDestination(const CTxDestination& dst) const; - bool IsUsedDestination(const uint256& hash, unsigned int n) const; - void SetUsedDestinationState(const uint256& hash, unsigned int n, bool used); + // Whether this or any known UTXO with the same single key has been spent. + bool IsUsedDestination(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; @@ -1022,39 +837,15 @@ public: int64_t ScanningDuration() const { return fScanningWallet ? GetTimeMillis() - m_scanning_start : 0; } double ScanningProgress() const { return fScanningWallet ? (double) m_scanning_progress : 0; } - /** - * keystore implementation - * Generate a new key - */ - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a key to the store, without saving it to disk (used by LoadWallet) - bool LoadKey(const CKey& key, const CPubKey &pubkey) { return AddKeyPubKeyInner(key, pubkey); } - //! Load metadata (used by LoadWallet) - void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } - void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - //! Adds an encrypted key to the store, and saves it to disk. - bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) - bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - bool GetKey(const CKeyID &address, CKey& keyOut) const override; - bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - bool HaveKey(const CKeyID &address) const override; - std::set<CKeyID> GetKeys() const override; - bool AddCScript(const CScript& redeemScript) override; - bool LoadCScript(const CScript& redeemScript); //! Adds a destination data tuple to the store, and saves it to disk - bool AddDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Erases a destination data tuple in the store and on disk - bool EraseDestData(const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool EraseDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a destination data tuple to the store, without saving it to disk void LoadDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Look up a destination data tuple in the store, return true if found false otherwise @@ -1062,18 +853,6 @@ public: //! Get all destination values matching a prefix. std::vector<std::string> GetDestValues(const std::string& prefix) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) - bool LoadWatchOnly(const CScript &dest); - //! Returns whether the watch-only script is in the wallet - bool HaveWatchOnly(const CScript &dest) const; - //! Returns whether there are any watch-only things in the wallet - bool HaveWatchOnly() const; - //! Fetches a pubkey from mapWatchKeys if it exists there - bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; - //! Holds a timestamp at which point the wallet is scheduled (externally) to be relocked. Caller must arrange for actual relocking to occur via Lock(). int64_t nRelockTime = 0; @@ -1095,8 +874,8 @@ public: bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); void LoadToWallet(CWalletTx& wtxIn) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void TransactionAddedToMempool(const CTransactionRef& tx) override; - void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const CBlock& block) override; + void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted, int height) override; + void BlockDisconnected(const CBlock& block, int height) override; void UpdatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); @@ -1117,7 +896,7 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; - void ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); struct Balance { CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more @@ -1151,9 +930,9 @@ public: * Should be called after CreateTransaction unless you want to abort * broadcasting the transaction. * - * @param tx[in] The transaction to be broadcast. - * @param mapValue[in] key-values to be set on the transaction. - * @param orderForm[in] BIP 70 / BIP 21 order form details to be set on the transaction. + * @param[in] tx The transaction to be broadcast. + * @param[in] mapValue key-values to be set on the transaction. + * @param[in] orderForm BIP 70 / BIP 21 order form details to be set on the transaction. */ void CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm); @@ -1189,42 +968,27 @@ public: /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; - bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool TopUpKeyPool(unsigned int kpSize = 0); - /** - * Reserves a key from the keypool and sets nIndex to its index - * - * @param[out] nIndex the index of the key in keypool - * @param[out] keypool the keypool the key was drawn from, which could be the - * the pre-split pool if present, or the internal or external pool - * @param fRequestedInternal true if the caller would like the key drawn - * from the internal keypool, false if external is preferred - * - * @return true if succeeded, false if failed due to empty keypool - * @throws std::runtime_error if keypool read failed, key was invalid, - * was not found in the wallet, or was misclassified in the internal - * or external keypool - */ - bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); - void KeepKey(int64_t nIndex); - void ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey); int64_t GetOldestKeyPoolTime(); - /** - * Marks all keys in the keypool up to and including reserve_key as used. - */ - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set<std::set<CTxDestination>> GetAddressGroupings() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::map<CTxDestination, CAmount> GetAddressBalances(interfaces::Chain::Lock& locked_chain); std::set<CTxDestination> GetLabelAddresses(const std::string& label) const; + /** + * Marks all outputs in each one of the destinations dirty, so their cache is + * reset and does not return outdated information. + */ + void MarkDestinationsDirty(const std::set<CTxDestination>& destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error); + isminetype IsMine(const CTxDestination& dest) const; + isminetype IsMine(const CScript& script) const; isminetype IsMine(const CTxIn& txin) const; /** * Returns amount of debit if the input matches the @@ -1254,14 +1018,10 @@ public: bool DelAddressBook(const CTxDestination& address); - unsigned int GetKeyPoolSize() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) - { - AssertLockHeld(cs_wallet); - return setInternalKeyPool.size() + setExternalKeyPool.size(); - } + unsigned int GetKeyPoolSize() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! signify that a particular wallet feature is now used. this may change nWalletVersion and nWalletMaxVersion if those are lower - void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false); + void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false) override; //! change which version we're allowed to upgrade to (note that this does not immediately imply upgrading to that format) bool SetMaxVersion(int nVersion); @@ -1321,7 +1081,7 @@ public: bool TransactionCanBeAbandoned(const uint256& hashTx) const; /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ - bool AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx); + bool AbandonTransaction(const uint256& hashTx); /** Mark a transaction as replaced by another transaction (e.g., BIP 125). */ bool MarkReplaced(const uint256& originalHash, const uint256& newHash); @@ -1340,31 +1100,12 @@ public: bool BackupWallet(const std::string& strDest); - /* Set the HD chain model (chain child index counters) */ - void SetHDChain(const CHDChain& chain, bool memonly); - const CHDChain& GetHDChain() const { return hdChain; } - /* Returns true if HD is enabled */ bool IsHDEnabled() const; - /* Returns true if the wallet can generate new keys */ - bool CanGenerateKeys(); - /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ bool CanGetAddresses(bool internal = false); - /* Generates a new HD seed (will not be activated) */ - CPubKey GenerateNewSeed(); - - /* Derives a new HD seed (will not be activated) */ - CPubKey DeriveNewSeed(const CKey& key); - - /* Set the current HD seed (will reset the chain child index counters) - Sets the seed's version based on the current wallet version (so the - caller must ensure the current wallet version is correct before calling - this function). */ - void SetHDSeed(const CPubKey& key); - /** * Blocks until the wallet state is up-to-date to /at least/ the current * chain at the time this function is entered @@ -1373,20 +1114,6 @@ public: */ void BlockUntilSyncedToCurrentChain() LOCKS_EXCLUDED(cs_main, cs_wallet); - /** - * Explicitly make the wallet learn the related scripts for outputs to the - * given key. This is purely to make the wallet file compatible with older - * software, as FillableSigningProvider automatically does this implicitly for all - * keys now. - */ - void LearnRelatedScripts(const CPubKey& key, OutputType); - - /** - * Same as LearnRelatedScripts, but when the OutputType is not known (and could - * be anything). - */ - void LearnAllRelatedScripts(const CPubKey& key); - /** set a single wallet flag */ void SetWalletFlag(uint64_t flags); @@ -1394,14 +1121,14 @@ public: void UnsetWalletFlag(uint64_t flag); /** check if a certain wallet flag is set */ - bool IsWalletFlagSet(uint64_t flag) const; + bool IsWalletFlagSet(uint64_t flag) const override; /** overwrite all flags by the given uint64_t returns false if unknown, non-tolerable flags are present */ bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); /** Returns a bracketed wallet name for displaying in logs, will return [default wallet] if the wallet has no name */ - const std::string GetDisplayName() const { + const std::string GetDisplayName() const override { std::string wallet_name = GetName().length() == 0 ? "default wallet" : GetName(); return strprintf("[%s]", wallet_name); }; @@ -1412,8 +1139,51 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; - /** Implement lookup of key origin information through wallet key metadata. */ - bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers + std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; + + //! Returns all unique ScriptPubKeyMans + std::set<ScriptPubKeyMan*> GetAllScriptPubKeyMans() const; + + //! Get the ScriptPubKeyMan for the given OutputType and internal/external chain. + ScriptPubKeyMan* GetScriptPubKeyMan(const OutputType& type, bool internal) const; + + //! Get the ScriptPubKeyMan for a script + ScriptPubKeyMan* GetScriptPubKeyMan(const CScript& script) const; + //! Get the ScriptPubKeyMan by id + ScriptPubKeyMan* GetScriptPubKeyMan(const uint256& id) const; + + //! Get the SigningProvider for a script + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script, SignatureData& sigdata) const; + + //! Get the LegacyScriptPubKeyMan which is used for all types, internal, and external. + LegacyScriptPubKeyMan* GetLegacyScriptPubKeyMan() const; + LegacyScriptPubKeyMan* GetOrCreateLegacyScriptPubKeyMan(); + + //! Make a LegacyScriptPubKeyMan and set it for all types, internal, and external. + void SetupLegacyScriptPubKeyMan(); + + const CKeyingMaterial& GetEncryptionKey() const override; + bool HasEncryptionKeys() const override; + + /** Get last block processed height */ + int GetLastBlockHeight() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + assert(m_last_block_processed_height >= 0); + return m_last_block_processed_height; + }; + /** Set last block processed height, currently only use in unit test */ + void SetLastBlockProcessed(int block_height, uint256 block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + m_last_block_processed_height = block_height; + m_last_block_processed = block_hash; + }; + + //! Connect the signals from ScriptPubKeyMans to the signals in CWallet + void ConnectScriptPubKeyManNotifiers(); }; /** diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index a9e6763c6d..a1928f45c4 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -250,8 +250,9 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssKey >> script; char fYes; ssValue >> fYes; - if (fYes == '1') - pwallet->LoadWatchOnly(script); + if (fYes == '1') { + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadWatchOnly(script); + } } else if (strType == DBKeys::KEY) { CPubKey vchPubKey; ssKey >> vchPubKey; @@ -302,12 +303,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strErr = "Error reading wallet database: CPrivKey corrupt"; return false; } - if (!pwallet->LoadKey(key, vchPubKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) { - strErr = "Error reading wallet database: LoadKey failed"; + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadKey failed"; return false; } } else if (strType == DBKeys::MASTER_KEY) { + // Master encryption key is loaded into only the wallet and not any of the ScriptPubKeyMans. unsigned int nID; ssKey >> nID; CMasterKey kMasterKey; @@ -332,9 +334,9 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> vchPrivKey; wss.nCKeys++; - if (!pwallet->LoadCryptedKey(vchPubKey, vchPrivKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) { - strErr = "Error reading wallet database: LoadCryptedKey failed"; + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; } wss.fIsEncrypted = true; @@ -344,14 +346,14 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->LoadScriptMetadata(CScriptID(script), keyMeta); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); } else if (strType == DBKeys::DEFAULTKEY) { // We don't want or need the default key, but if there is one set, // we want to make sure that it is valid so that we can detect corruption @@ -367,15 +369,15 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyPool keypool; ssValue >> keypool; - pwallet->LoadKeyPool(nIndex, keypool); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); } else if (strType == DBKeys::CSCRIPT) { uint160 hash; ssKey >> hash; CScript script; ssValue >> script; - if (!pwallet->LoadCScript(script)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCScript(script)) { - strErr = "Error reading wallet database: LoadCScript failed"; + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCScript failed"; return false; } } else if (strType == DBKeys::ORDERPOSNEXT) { @@ -389,7 +391,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; - pwallet->SetHDChain(chain, true); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->SetHDChain(chain, true); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; @@ -512,8 +514,13 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); // nTimeFirstKey is only reliable if all keys have metadata - if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) - pwallet->UpdateTimeFirstKey(1); + if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { + auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); + if (spk_man) { + LOCK(spk_man->cs_KeyStore); + spk_man->UpdateTimeFirstKey(1); + } + } for (const uint256& hash : wss.vWalletUpgrade) WriteTx(pwallet->mapWallet.at(hash)); @@ -622,7 +629,7 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u } else if ((*it) == hash) { if(!EraseTx(hash)) { - LogPrint(BCLog::DB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); + LogPrint(BCLog::WALLETDB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); delerror = true; } vTxHashOut.push_back(hash); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index b1781d5ccf..1a65125480 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -124,7 +124,7 @@ public: std::string hdKeypath; //optional HD/bip32 keypath. Still used to determine whether a key is a seed. Also kept for backwards compatibility CKeyID hd_seed_id; //id of the HD seed used to derive this key KeyOriginInfo key_origin; // Key origin info with path and fingerprint - bool has_key_origin = false; //< Whether the key_origin is useful + bool has_key_origin = false; //!< Whether the key_origin is useful CKeyMetadata() { diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 0843194511..fbfdf9dd6b 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -27,18 +27,20 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: } // dummy chain interface std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + LOCK(wallet_instance->cs_wallet); bool first_run = true; DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); if (load_wallet_ret != DBErrors::LOAD_OK) { - tfm::format(std::cerr, "Error creating %s", name.c_str()); + tfm::format(std::cerr, "Error creating %s", name); return nullptr; } wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); // generate a new HD seed - CPubKey seed = wallet_instance->GenerateNewSeed(); - wallet_instance->SetHDSeed(seed); + auto spk_man = wallet_instance->GetOrCreateLegacyScriptPubKeyMan(); + CPubKey seed = spk_man->GenerateNewSeed(); + spk_man->SetHDSeed(seed); tfm::format(std::cout, "Topping up keypool...\n"); wallet_instance->TopUpKeyPool(); @@ -59,28 +61,28 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa bool first_run; load_wallet_ret = wallet_instance->LoadWallet(first_run); } catch (const std::runtime_error&) { - tfm::format(std::cerr, "Error loading %s. Is wallet being used by another process?\n", name.c_str()); + tfm::format(std::cerr, "Error loading %s. Is wallet being used by another process?\n", name); return nullptr; } if (load_wallet_ret != DBErrors::LOAD_OK) { wallet_instance = nullptr; if (load_wallet_ret == DBErrors::CORRUPT) { - tfm::format(std::cerr, "Error loading %s: Wallet corrupted", name.c_str()); + tfm::format(std::cerr, "Error loading %s: Wallet corrupted", name); return nullptr; } else if (load_wallet_ret == DBErrors::NONCRITICAL_ERROR) { tfm::format(std::cerr, "Error reading %s! All keys read correctly, but transaction data" " or address book entries might be missing or incorrect.", - name.c_str()); + name); } else if (load_wallet_ret == DBErrors::TOO_NEW) { tfm::format(std::cerr, "Error loading %s: Wallet requires newer version of %s", - name.c_str(), PACKAGE_NAME); + name, PACKAGE_NAME); return nullptr; } else if (load_wallet_ret == DBErrors::NEED_REWRITE) { tfm::format(std::cerr, "Wallet needed to be rewritten: restart %s to complete", PACKAGE_NAME); return nullptr; } else { - tfm::format(std::cerr, "Error loading %s", name.c_str()); + tfm::format(std::cerr, "Error loading %s", name); return nullptr; } } @@ -94,7 +96,7 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Wallet info\n===========\n"); tfm::format(std::cout, "Encrypted: %s\n", wallet_instance->IsCrypted() ? "yes" : "no"); - tfm::format(std::cout, "HD (hd seed available): %s\n", wallet_instance->GetHDChain().seed_id.IsNull() ? "no" : "yes"); + tfm::format(std::cout, "HD (hd seed available): %s\n", wallet_instance->IsHDEnabled() ? "yes" : "no"); tfm::format(std::cout, "Keypool Size: %u\n", wallet_instance->GetKeyPoolSize()); tfm::format(std::cout, "Transactions: %zu\n", wallet_instance->mapWallet.size()); tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->mapAddressBook.size()); @@ -112,12 +114,12 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) } } else if (command == "info") { if (!fs::exists(path)) { - tfm::format(std::cerr, "Error: no wallet file at %s\n", name.c_str()); + tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } std::string error; if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "Error loading %s. Is wallet being used by other process?\n", name.c_str()); + tfm::format(std::cerr, "Error loading %s. Is wallet being used by other process?\n", name); return false; } std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); @@ -125,7 +127,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) WalletShowInfo(wallet_instance.get()); wallet_instance->Flush(true); } else { - tfm::format(std::cerr, "Invalid command: %s\n", command.c_str()); + tfm::format(std::cerr, "Invalid command: %s\n", command); return false; } diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index bd08da42d6..8ee3355f02 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index ba2f913841..c91c9aca96 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,54 @@ #include <vector> +/** (client) version numbers for particular wallet features */ +enum WalletFeature +{ + FEATURE_BASE = 10500, // the earliest version new wallets supports (only useful for getwalletinfo's clientversion output) + + FEATURE_WALLETCRYPT = 40000, // wallet encryption + FEATURE_COMPRPUBKEY = 60000, // compressed public keys + + FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) + + FEATURE_HD_SPLIT = 139900, // Wallet with HD chain split (change outputs will use m/0'/1'/k) + + FEATURE_NO_DEFAULT_KEY = 159900, // Wallet without a default key written + + FEATURE_PRE_SPLIT_KEYPOOL = 169900, // Upgraded to HD SPLIT and can have a pre-split keypool + + FEATURE_LATEST = FEATURE_PRE_SPLIT_KEYPOOL +}; + + + +enum WalletFlags : uint64_t { + // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown + // unknown wallet flags in the lower section <= (1 << 31) will be tolerated + + // will categorize coins as clean (not reused) and dirty (reused), and handle + // them with privacy considerations in mind + WALLET_FLAG_AVOID_REUSE = (1ULL << 0), + + // Indicates that the metadata has already been upgraded to contain key origins + WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), + + // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) + WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), + + //! Flag set when a wallet contains no HD seed and no private keys, scripts, + //! addresses, and other watch only things, and is therefore "blank." + //! + //! The only function this flag serves is to distinguish a blank wallet from + //! a newly created wallet when the wallet database is loaded, to avoid + //! initialization that should only happen on first run. + //! + //! This flag is also a mandatory flag to prevent previous versions of + //! bitcoin from opening the wallet, thinking it was newly created, and + //! then improperly reinitializing it. + WALLET_FLAG_BLANK_WALLET = (1ULL << 33), +}; + //! Get the path of the wallet directory. fs::path GetWalletDir(); diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h index 2e1fdf4f3a..f4730273f1 100644 --- a/src/walletinitinterface.h +++ b/src/walletinitinterface.h @@ -1,11 +1,11 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLETINITINTERFACE_H #define BITCOIN_WALLETINITINTERFACE_H -struct InitInterfaces; +struct NodeContext; class WalletInitInterface { public: @@ -15,8 +15,8 @@ public: virtual void AddWalletOptions() const = 0; /** Check wallet parameter interaction */ virtual bool ParameterInteraction() const = 0; - /** Add wallets that should be opened to list of init interfaces. */ - virtual void Construct(InitInterfaces& interfaces) const = 0; + /** Add wallets that should be opened to list of chain clients. */ + virtual void Construct(NodeContext& node) const = 0; virtual ~WalletInitInterface() {} }; diff --git a/src/warnings.cpp b/src/warnings.cpp index 35d2033ba8..467c3d0f65 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,41 +38,34 @@ void SetfLargeWorkInvalidChainFound(bool flag) fLargeWorkInvalidChainFound = flag; } -std::string GetWarnings(const std::string& strFor) +std::string GetWarnings(bool verbose) { - std::string strStatusBar; - std::string strGUI; - const std::string uiAlertSeperator = "<hr />"; + std::string warnings_concise; + std::string warnings_verbose; + const std::string warning_separator = "<hr />"; LOCK(cs_warnings); + // Pre-release build warning if (!CLIENT_VERSION_IS_RELEASE) { - strStatusBar = "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"; - strGUI = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications").translated; + warnings_concise = "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"; + warnings_verbose = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications").translated; } // Misc warnings like out of disk space and clock is wrong - if (strMiscWarning != "") - { - strStatusBar = strMiscWarning; - strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + strMiscWarning; + if (strMiscWarning != "") { + warnings_concise = strMiscWarning; + warnings_verbose += (warnings_verbose.empty() ? "" : warning_separator) + strMiscWarning; } - if (fLargeWorkForkFound) - { - strStatusBar = "Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."; - strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues.").translated; - } - else if (fLargeWorkInvalidChainFound) - { - strStatusBar = "Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."; - strGUI += (strGUI.empty() ? "" : uiAlertSeperator) + _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.").translated; + if (fLargeWorkForkFound) { + warnings_concise = "Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."; + warnings_verbose += (warnings_verbose.empty() ? "" : warning_separator) + _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues.").translated; + } else if (fLargeWorkInvalidChainFound) { + warnings_concise = "Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."; + warnings_verbose += (warnings_verbose.empty() ? "" : warning_separator) + _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.").translated; } - if (strFor == "gui") - return strGUI; - else if (strFor == "statusbar") - return strStatusBar; - assert(!"GetWarnings(): invalid parameter"); - return "error"; + if (verbose) return warnings_verbose; + else return warnings_concise; } diff --git a/src/warnings.h b/src/warnings.h index e6701ebd9e..83b1add1ee 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,11 +13,11 @@ void SetfLargeWorkForkFound(bool flag); bool GetfLargeWorkForkFound(); void SetfLargeWorkInvalidChainFound(bool flag); /** Format a string that describes several potential problems detected by the core. - * @param[in] strFor can have the following values: - * - "statusbar": get the most important warning - * - "gui": get all warnings, translated (where possible) for GUI, separated by <hr /> - * @returns the warning string selected by strFor + * @param[in] verbose bool + * - if true, get all warnings, translated (where possible), separated by <hr /> + * - if false, get the most important warning + * @returns the warning string */ -std::string GetWarnings(const std::string& strFor); +std::string GetWarnings(bool verbose); #endif // BITCOIN_WARNINGS_H diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp index a5f3be8f5b..aae760adde 100644 --- a/src/zmq/zmqabstractnotifier.cpp +++ b/src/zmq/zmqabstractnotifier.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h index e3fdbf7402..5f0036206d 100644 --- a/src/zmq/zmqconfig.h +++ b/src/zmq/zmqconfig.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index ebbaf8683d..0ce14f232e 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -185,7 +185,7 @@ void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBloc } } -void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) +void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) { for (const CTransactionRef& ptx : pblock->vtx) { // Do a normal notify for each transaction removed in block disconnection diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 6be0554a65..c820865497 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,7 +27,7 @@ protected: // CValidationInterface void TransactionAddedToMempool(const CTransactionRef& tx) override; void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; private: diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index ba89d1401d..04806903c2 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -112,7 +112,8 @@ bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) void CZMQAbstractPublishNotifier::Shutdown() { - assert(psocket); + // Early return if Initialize was not called + if (!psocket) return; int count = mapPublishNotifiers.count(address); diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index cf97b7ecce..5652877f3c 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/README.md b/test/README.md index 11adc11278..c3e4ae9ad2 100644 --- a/test/README.md +++ b/test/README.md @@ -254,7 +254,14 @@ Use the `-v` option for verbose output. #### Dependencies -The lint tests require codespell and flake8. To install: `pip3 install codespell flake8`. +| Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation +|-----------|:----------:|:-------------------------------------------:|-------------- +| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8` +| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing) +| [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq` +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0` + +Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. #### Running the tests diff --git a/test/config.ini.in b/test/config.ini.in index 060c553da2..9687206ee1 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -16,6 +16,7 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py # Which components are enabled. These are commented out by `configure` if they were disabled when running config. @ENABLE_WALLET_TRUE@ENABLE_WALLET=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true +@BUILD_BITCOIN_WALLET_TRUE@ENABLE_WALLET_TOOL=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true diff --git a/test/functional/README.md b/test/functional/README.md index a9b83076eb..77a9ce9acb 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -99,6 +99,16 @@ P2PInterface object and override the callback methods. Examples tests are [p2p_unrequested_blocks.py](p2p_unrequested_blocks.py), [p2p_compactblocks.py](p2p_compactblocks.py). +#### Prototyping tests + +The [`TestShell`](test-shell.md) class exposes the BitcoinTestFramework +functionality to interactive Python3 environments and can be used to prototype +tests. This may be especially useful in a REPL environment with session logging +utilities, such as +[IPython](https://ipython.readthedocs.io/en/stable/interactive/reference.html#session-logging-and-restoring). +The logs of such interactive sessions can later be adapted into permanent test +cases. + ### Test framework modules The following are useful modules for test developers. They are located in [test/functional/test_framework/](test_framework). diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index a70d9c4ac1..00f2833f55 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. """Combine logs from multiple bitcoin nodes as well as the test_framework log. This streams the combined log output to stdout. Use combine_logs.py > outputfile diff --git a/test/functional/create_cache.py b/test/functional/create_cache.py index edf16fa47e..1108a8e354 100755 --- a/test/functional/create_cache.py +++ b/test/functional/create_cache.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Create a blockchain cache. @@ -16,7 +16,6 @@ class CreateCache(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 0 - self.supports_cli = True def setup_network(self): pass diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index fd69bbd2c7..99c88bbcc0 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 62c3eca07d..9b878e8bf8 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -29,7 +29,7 @@ class AbortNodeTest(BitcoinTestFramework): datadir = get_datadir_path(self.options.tmpdir, 0) # Deleting the undo file will result in reorg failure - os.unlink(os.path.join(datadir, 'regtest', 'blocks', 'rev00000.dat')) + os.unlink(os.path.join(datadir, self.chain, 'blocks', 'rev00000.dat')) # Connecting to a node with a more work chain will trigger a reorg # attempt. @@ -40,7 +40,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=60) + wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=200) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index c7e98bd4db..95905f477b 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1401,7 +1401,7 @@ class FullBlockTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() self.bootstrap_p2p(timeout=timeout) - def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): + def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=960): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index 6f01f97ea2..7bfad52c24 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the blocksdir option. diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 7d131e6045..2c6f2e733b 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -135,7 +135,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputs on {} failed with non-mandatory-script-verify-flag (Negative locktime)'.format(block.vtx[-1].hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Negative locktime)'.format(block.vtx[-1].hash)]): self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) self.nodes[0].p2p.sync_with_ping() diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index b997c76025..719b124a65 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -13,6 +13,7 @@ class ConfArgsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + self.supports_cli = False def test_config_file_parser(self): # Assume node is stopped @@ -38,7 +39,7 @@ class ConfArgsTest(BitcoinTestFramework): if self.is_wallet_compiled(): with open(inc_conf_file_path, 'w', encoding='utf8') as conf: conf.write("wallet=foo\n") - self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on regtest network when in [regtest] section.') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on %s network when in [%s] section.' % (self.chain, self.chain)) with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('regtest=0\n') # mainnet @@ -78,14 +79,44 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('') # clear def test_log_buffer(self): - with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0']): + with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0\n']): self.start_node(0, extra_args=['-noconnect=0']) self.stop_node(0) + def test_args_log(self): + self.log.info('Test config args logging') + with self.nodes[0].assert_debug_log( + expected_msgs=[ + 'Command-line arg: addnode="some.node"', + 'Command-line arg: rpcauth=****', + 'Command-line arg: rpcbind=****', + 'Command-line arg: rpcpassword=****', + 'Command-line arg: rpcuser=****', + 'Command-line arg: torpassword=****', + 'Config file arg: regtest="1"', + 'Config file arg: [regtest] server="1"', + ], + unexpected_msgs=[ + 'alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', + '127.1.1.1', + 'secret-rpcuser', + 'secret-torpassword', + ]): + self.start_node(0, extra_args=[ + '-addnode=some.node', + '-rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', + '-rpcbind=127.1.1.1', + '-rpcpassword=', + '-rpcuser=secret-rpcuser', + '-torpassword=secret-torpassword', + ]) + self.stop_node(0) + def run_test(self): self.stop_node(0) self.test_log_buffer() + self.test_args_log() self.test_config_file_parser() @@ -103,7 +134,7 @@ class ConfArgsTest(BitcoinTestFramework): # Check that using non-existent datadir in conf file fails conf_file = os.path.join(default_data_dir, "bitcoin.conf") - # datadir needs to be set before [regtest] section + # datadir needs to be set before [chain] section conf_file_contents = open(conf_file, encoding='utf8').read() with open(conf_file, 'w', encoding='utf8') as f: f.write("datadir=" + new_data_dir + "\n") @@ -115,17 +146,17 @@ class ConfArgsTest(BitcoinTestFramework): os.mkdir(new_data_dir) self.start_node(0, ['-conf='+conf_file, '-wallet=w1']) self.stop_node(0) - assert os.path.exists(os.path.join(new_data_dir, 'regtest', 'blocks')) + assert os.path.exists(os.path.join(new_data_dir, self.chain, 'blocks')) if self.is_wallet_compiled(): - assert os.path.exists(os.path.join(new_data_dir, 'regtest', 'wallets', 'w1')) + assert os.path.exists(os.path.join(new_data_dir, self.chain, 'wallets', 'w1')) # Ensure command line argument overrides datadir in conf os.mkdir(new_data_dir_2) self.nodes[0].datadir = new_data_dir_2 self.start_node(0, ['-datadir='+new_data_dir_2, '-conf='+conf_file, '-wallet=w2']) - assert os.path.exists(os.path.join(new_data_dir_2, 'regtest', 'blocks')) + assert os.path.exists(os.path.join(new_data_dir_2, self.chain, 'blocks')) if self.is_wallet_compiled(): - assert os.path.exists(os.path.join(new_data_dir_2, 'regtest', 'wallets', 'w2')) + assert os.path.exists(os.path.join(new_data_dir_2, self.chain, 'wallets', 'w2')) if __name__ == '__main__': diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 6bd321992a..c2b4de54f2 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -139,6 +139,7 @@ class BIP68_112_113Test(BitcoinTestFramework): self.num_nodes = 1 self.setup_clean_chain = True self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=4', '-addresstype=legacy']] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 6bd6bb5b8c..5bbdb8cda1 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -51,6 +51,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.num_nodes = 4 self.setup_clean_chain = False self.rpc_timeout = 480 + self.supports_cli = False # Set -maxmempool=0 to turn off mempool memory sharing with dbcache # Set -rpcservertimeout=900 to reduce socket disconnects in this diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 2ace96fef4..27da49cf24 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -120,7 +120,7 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputs on {} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)'.format(block.vtx[-1].hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)'.format(block.vtx[-1].hash)]): self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) self.nodes[0].p2p.sync_with_ping() diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index ba116e41f5..be1c044aa5 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" @@ -19,7 +19,7 @@ class FilelockTest(BitcoinTestFramework): self.nodes[0].wait_for_rpc_connection() def run_test(self): - datadir = os.path.join(self.nodes[0].datadir, 'regtest') + datadir = os.path.join(self.nodes[0].datadir, self.chain) self.log.info("Using datadir {}".format(datadir)) self.log.info("Check that we can't start a second bitcoind instance using the same datadir") diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index ed1d25c0d6..e3e2456183 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -17,7 +17,7 @@ class HelpTest(BitcoinTestFramework): # Don't start the node def get_node_output(self, *, ret_code_expected): - ret_code = self.nodes[0].process.wait(timeout=5) + ret_code = self.nodes[0].process.wait(timeout=60) assert_equal(ret_code, ret_code_expected) self.nodes[0].stdout.seek(0) self.nodes[0].stderr.seek(0) diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 2cd6a05d08..6f1a0cd348 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests the includeconf argument diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index bf2a4ff61f..7f1e8f5bae 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -26,6 +26,7 @@ class LoadblockTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 + self.supports_cli = False def run_test(self): self.nodes[1].setnetworkactive(state=False) @@ -37,7 +38,7 @@ class LoadblockTest(BitcoinTestFramework): cfg_file = os.path.join(data_dir, "linearize.cfg") bootstrap_file = os.path.join(self.options.tmpdir, "bootstrap.dat") genesis_block = self.nodes[0].getblockhash(0) - blocks_dir = os.path.join(data_dir, "regtest", "blocks") + blocks_dir = os.path.join(data_dir, self.chain, "blocks") hash_list = tempfile.NamedTemporaryFile(dir=data_dir, mode='w', delete=False, diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index e6ff21ee9c..1bae29b302 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test debug logging.""" @@ -16,7 +16,7 @@ class LoggingTest(BitcoinTestFramework): self.setup_clean_chain = True def relative_log_path(self, name): - return os.path.join(self.nodes[0].datadir, "regtest", name) + return os.path.join(self.nodes[0].datadir, self.chain, name) def run_test(self): # test default log file name diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 180ea0e51d..8d200915bd 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of -maxuploadtarget. @@ -36,6 +36,7 @@ class MaxUploadTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [["-maxuploadtarget=800", "-acceptnonstdtxn=1"]] + self.supports_cli = False # Cache for utxos, as the listunspent may take a long time later in the test self.utxo_cache = [] diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 51523f13e7..21b6b299f6 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -78,6 +78,7 @@ class PruneTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 6 + self.supports_cli = False # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. @@ -100,7 +101,7 @@ class PruneTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - self.prunedir = os.path.join(self.nodes[2].datadir, 'regtest', 'blocks', '') + self.prunedir = os.path.join(self.nodes[2].datadir, self.chain, 'blocks', '') connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) @@ -278,7 +279,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(ret, node.getblockchaininfo()['pruneheight']) def has_block(index): - return os.path.isfile(os.path.join(self.nodes[node_number].datadir, "regtest", "blocks", "blk{:05}.dat".format(index))) + return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", "blk{:05}.dat".format(index))) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index e7afbd0272..9e578f0026 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -76,6 +76,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): "-limitdescendantsize=101", ], ] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py index 5084cb1322..d782d3b1d8 100755 --- a/test/functional/feature_shutdown.py +++ b/test/functional/feature_shutdown.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind shutdown.""" @@ -17,6 +17,7 @@ class ShutdownTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + self.supports_cli = False def run_test(self): node = get_rpc_proxy(self.nodes[0].url, 1, timeout=600, coveragedir=self.nodes[0].coverage_dir) diff --git a/test/functional/feature_uacomment.py b/test/functional/feature_uacomment.py index 85c250173f..4720f6dea3 100755 --- a/test/functional/feature_uacomment.py +++ b/test/functional/feature_uacomment.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -uacomment option.""" diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 0a378c5ef5..f04a58cd19 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -12,6 +12,9 @@ class TestBitcoinCli(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 + def skip_test_if_missing_module(self): + self.skip_if_no_cli() + def run_test(self): """Main test logic""" @@ -56,7 +59,6 @@ class TestBitcoinCli(BitcoinTestFramework): blockchain_info = self.nodes[0].getblockchaininfo() assert_equal(cli_get_info['version'], network_info['version']) - assert_equal(cli_get_info['protocolversion'], network_info['protocolversion']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) @@ -64,9 +66,7 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): - assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) assert_equal(cli_get_info['balance'], wallet_info['balance']) - assert_equal(cli_get_info['keypoololdest'], wallet_info['keypoololdest']) assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index bb868d7115..d007490f80 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -13,6 +13,7 @@ import urllib.parse class HTTPBasicsTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 + self.supports_cli = False def setup_network(self): self.setup_nodes() diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index a036dfc790..e73ec90819 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -44,6 +44,7 @@ class RESTTest (BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-rest"], []] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -151,7 +152,7 @@ class RESTTest (BitcoinTestFramework): bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) output = BytesIO(bin_response) - chain_height, = unpack("i", output.read(4)) + chain_height, = unpack("<i", output.read(4)) response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index e99fa22646..3b3b5bb0c1 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -22,6 +22,7 @@ class RPCInterfaceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + self.supports_cli = False def test_getrpcinfo(self): self.log.info("Testing getrpcinfo...") @@ -32,7 +33,7 @@ class RPCInterfaceTest(BitcoinTestFramework): command = info['active_commands'][0] assert_equal(command['method'], 'getrpcinfo') assert_greater_than_or_equal(command['duration'], 0) - assert_equal(info['logpath'], os.path.join(self.nodes[0].datadir, 'regtest', 'debug.log')) + assert_equal(info['logpath'], os.path.join(self.nodes[0].datadir, self.chain, 'debug.log')) def test_batch_request(self): self.log.info("Testing basic JSON-RPC batch request...") diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 5aea10fbce..89c55f31f3 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -59,6 +59,10 @@ class ZMQTest (BitcoinTestFramework): # Note that the publishing order is not defined in the documentation and # is subject to change. import zmq + + # Invalid zmq arguments don't take down the node, see #17185. + self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) + address = 'tcp://127.0.0.1:28332' socket = self.ctx.socket(zmq.SUB) socket.set(zmq.RCVTIMEO, 60000) diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index c466b120f2..900cabccda 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -8,6 +8,7 @@ from io import BytesIO import math from test_framework.test_framework import BitcoinTestFramework +from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, @@ -20,6 +21,9 @@ from test_framework.script import ( hash160, CScript, OP_0, + OP_2, + OP_3, + OP_CHECKMULTISIG, OP_EQUAL, OP_HASH160, OP_RETURN, @@ -35,8 +39,9 @@ class MempoolAcceptanceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [[ - '-txindex', + '-txindex','-permitbaremultisig=0', ]] * self.num_nodes + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -261,12 +266,27 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() + tx.vout[0].scriptPubKey = CScript([OP_2, pubkey, pubkey, pubkey, OP_3, OP_CHECKMULTISIG]) # Some bare multisig script (2-of-3) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bare-multisig'}], + rawtxs=[tx.serialize().hex()], + ) + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'scriptsig-not-pushonly'}], rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) + tx.vin[0].scriptSig = CScript([b'a' * 1648]) # Some too large scriptSig (>1650 bytes) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'scriptsig-size'}], + rawtxs=[tx.serialize().hex()], + ) + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) num_scripts = 100000 // len(output_p2sh_burn.serialize()) # Use enough outputs to make the tx too large for our policy tx.vout = [output_p2sh_burn] * num_scripts diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index edf2069933..39035f7cb1 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -18,6 +18,7 @@ class MempoolLimitTest(BitcoinTestFramework): "-maxmempool=5", "-spendzeroconfchange=0", ]] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index c7d241503a..7014105d88 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -14,13 +14,19 @@ from test_framework.util import ( satoshi_round, ) +# default limits MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 +# custom limits for node1 +MAX_ANCESTORS_CUSTOM = 5 class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.extra_args = [["-maxorphantx=1000"], ["-maxorphantx=1000", "-limitancestorcount=5"]] + self.extra_args = [ + ["-maxorphantx=1000"], + ["-maxorphantx=1000", "-limitancestorcount={}".format(MAX_ANCESTORS_CUSTOM)], + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -188,7 +194,14 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 2000) assert_equal(mempool[x]['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) - # TODO: check that node1's mempool is as expected + # Check that node1's mempool is as expected (-> custom ancestor limit) + mempool0 = self.nodes[0].getrawmempool(False) + mempool1 = self.nodes[1].getrawmempool(False) + assert_equal(len(mempool1), MAX_ANCESTORS_CUSTOM) + assert set(mempool1).issubset(set(mempool0)) + for tx in chain[:MAX_ANCESTORS_CUSTOM]: + assert tx in mempool1 + # TODO: more detailed check of node1's mempool (fees etc.) # TODO: test ancestor size limits diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 398fdeb326..d3690b245e 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. @@ -117,8 +117,8 @@ class MempoolPersistTest(BitcoinTestFramework): wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 5) - mempooldat0 = os.path.join(self.nodes[0].datadir, 'regtest', 'mempool.dat') - mempooldat1 = os.path.join(self.nodes[1].datadir, 'regtest', 'mempool.dat') + mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') + mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) self.nodes[0].savemempool() diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 123f0b4c28..3b148d5cf0 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -76,7 +76,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_101_id = self.nodes[0].sendrawtransaction(spend_101_raw) spend_102_1_id = self.nodes[0].sendrawtransaction(spend_102_1_raw) - self.sync_all() + self.sync_all(timeout=360) assert_equal(set(self.nodes[0].getrawmempool()), {spend_101_id, spend_102_1_id, timelock_tx_id}) @@ -91,7 +91,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): for node in self.nodes: node.invalidateblock(new_blocks[0]) - self.sync_all() + self.sync_all(timeout=360) # mempool should be empty. assert_equal(set(self.nodes[0].getrawmempool()), set()) diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index f9231614ce..8262e30592 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -43,6 +43,7 @@ class MiningTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + self.supports_cli = False def mine_chain(self): self.log.info('Create some old blocks') @@ -69,7 +70,7 @@ class MiningTest(BitcoinTestFramework): self.log.info('getmininginfo') mining_info = node.getmininginfo() assert_equal(mining_info['blocks'], 200) - assert_equal(mining_info['chain'], 'regtest') + assert_equal(mining_info['chain'], self.chain) assert 'currentblocktx' not in mining_info assert 'currentblockweight' not in mining_info assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10')) diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 445ec124ce..6d0b241e57 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -27,6 +27,7 @@ class LongpollThread(threading.Thread): class GetBlockTemplateLPTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 7e05a8e6c8..1426fdaacb 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -18,6 +18,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): "-printpriority=1", "-acceptnonstdtxn=1", ]] * self.num_nodes + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 23dea4b729..9047fc6828 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" @@ -16,6 +16,7 @@ from test_framework.util import ( class DisconnectBanTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.supports_cli = False def run_test(self): self.log.info("Connect nodes both way") diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 58c72f89d8..9876d749ff 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" import asyncio -import os import struct import sys @@ -66,27 +65,21 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit) assert len(msg_at_size.serialize()) == msg_limit - increase_allowed = 0.5 - if [s for s in os.environ.get("BITCOIN_CONFIG", "").split(" ") if "--with-sanitizers" in s and "address" in s]: - increase_allowed = 3.5 - with node.assert_memory_usage_stable(increase_allowed=increase_allowed): - self.log.info( - "Sending a bunch of large, junk messages to test " - "memory exhaustion. May take a bit...") + self.log.info("Sending a bunch of large, junk messages to test memory exhaustion. May take a bit...") - # Run a bunch of times to test for memory exhaustion. - for _ in range(80): - node.p2p.send_message(msg_at_size) + # Run a bunch of times to test for memory exhaustion. + for _ in range(80): + node.p2p.send_message(msg_at_size) - # Check that, even though the node is being hammered by nonsense from one - # connection, it can still service other peers in a timely way. - for _ in range(20): - conn2.sync_with_ping(timeout=2) + # Check that, even though the node is being hammered by nonsense from one + # connection, it can still service other peers in a timely way. + for _ in range(20): + conn2.sync_with_ping(timeout=2) - # Peer 1, despite serving up a bunch of nonsense, should still be connected. - self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=320) - assert node.p2p.is_connected + # Peer 1, despite serving up a bunch of nonsense, should still be connected. + self.log.info("Waiting for node to drop junk messages.") + node.p2p.sync_with_ping(timeout=400) + assert node.p2p.is_connected # # 1. @@ -101,11 +94,10 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_over_size = msg_unrecognized(str_data="b" * (valid_data_limit + 1)) assert len(msg_over_size.serialize()) == (msg_limit + 1) - with node.assert_debug_log(["Oversized message from peer=4, disconnecting"]): - # An unknown message type (or *any* message type) over - # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. - node.p2p.send_message(msg_over_size) - node.p2p.wait_for_disconnect(timeout=4) + # An unknown message type (or *any* message type) over + # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. + node.p2p.send_message(msg_over_size) + node.p2p.wait_for_disconnect(timeout=4) node.disconnect_p2ps() conn = node.add_p2p_connection(P2PDataStore()) @@ -153,13 +145,13 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_magic_bytes(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - def swap_magic_bytes(): + async def swap_magic_bytes(): conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes conn.magic_bytes = b'\x00\x11\x22\x32' # Call .result() to block until the atomic swap is complete, otherwise # we might run into races later on - asyncio.run_coroutine_threadsafe(asyncio.coroutine(swap_magic_bytes)(), NetworkThread.network_event_loop).result() + asyncio.run_coroutine_threadsafe(swap_magic_bytes(), NetworkThread.network_event_loop).result() with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): conn.send_message(messages.msg_ping(nonce=0xff)) @@ -168,7 +160,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_checksum(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - with self.nodes[0].assert_debug_log(['ProcessMessages(badmsg, 2 bytes): CHECKSUM ERROR expected 78df0a04 was ffffffff']): + with self.nodes[0].assert_debug_log(['CHECKSUM ERROR (badmsg, 2 bytes), expected 78df0a04 was ffffffff']): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 3cca2d78db..5975a52b2a 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid transactions. diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 40b28d7533..93e2957fd0 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p permission message. @@ -7,21 +7,35 @@ Test that permissions are correctly calculated and applied """ +from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.messages import ( + CTransaction, + CTxInWitness, + FromHex, +) +from test_framework.mininode import P2PDataStore +from test_framework.script import ( + CScript, + OP_TRUE, +) from test_framework.test_node import ErrorMatch from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, p2p_port, + wait_until, ) + class P2PPermissionsTests(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[],[]] def run_test(self): + self.check_tx_relay() + self.checkpermission( # default permissions (no specific permissions) ["-whitelist=127.0.0.1"], @@ -48,9 +62,9 @@ class P2PPermissionsTests(BitcoinTestFramework): ip_port = "127.0.0.1:{}".format(p2p_port(1)) self.replaceinconfig(1, "bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port) self.checkpermission( - ["-whitelist=noban@127.0.0.1" ], + ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay - ["noban", "bloomfilter", "forcerelay", "relay" ], + ["noban", "bloomfilter", "forcerelay", "relay"], False) self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") @@ -83,6 +97,41 @@ class P2PPermissionsTests(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(["-whitelist=noban@127.0.0.1:230"], "Invalid netmask specified in", match=ErrorMatch.PARTIAL_REGEX) self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1/10"], "Cannot resolve -whitebind address", match=ErrorMatch.PARTIAL_REGEX) + def check_tx_relay(self): + block_op_true = self.nodes[0].getblock(self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) + self.sync_all() + + self.log.debug("Create a connection from a whitelisted wallet that rebroadcasts raw txs") + # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only + # rebroadcast via the inv-getdata mechanism. However, even for whitelisted connections, a full node would + # currently not request a txid that is already in the mempool. + self.restart_node(1, extra_args=["-whitelist=forcerelay@127.0.0.1"]) + p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) + + self.log.debug("Send a tx from the wallet initially") + tx = FromHex( + CTransaction(), + self.nodes[0].createrawtransaction( + inputs=[{ + 'txid': block_op_true['tx'][0], + 'vout': 0, + }], outputs=[{ + ADDRESS_BCRT1_P2WSH_OP_TRUE: 5, + }]), + ) + tx.wit.vtxinwit = [CTxInWitness()] + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + txid = tx.rehash() + + self.log.debug("Wait until tx is in node[1]'s mempool") + p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) + + self.log.debug("Check that node[1] will send the tx to node[0] even though it is already in the mempool") + connect_nodes(self.nodes[1], 0) + with self.nodes[1].assert_debug_log(["Force relaying tx {} from whitelisted peer=0".format(txid)]): + p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) + wait_until(lambda: txid in self.nodes[0].getrawmempool()) + def checkpermission(self, args, expectedPermissions, whitelisted): self.restart_node(1, args) connect_nodes(self.nodes[0], 1) @@ -95,9 +144,10 @@ class P2PPermissionsTests(BitcoinTestFramework): def replaceinconfig(self, nodeid, old, new): with open(self.nodes[nodeid].bitcoinconf, encoding="utf8") as f: - newText=f.read().replace(old, new) + newText = f.read().replace(old, new) with open(self.nodes[nodeid].bitcoinconf, 'w', encoding="utf8") as f: f.write(newText) + if __name__ == '__main__': P2PPermissionsTests().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 0c7edbf434..3223c27e0b 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -31,7 +31,7 @@ from test_framework.messages import ( msg_inv, msg_tx, msg_block, - msg_witness_tx, + msg_no_witness_tx, ser_uint256, ser_vector, sha256, @@ -125,10 +125,11 @@ def test_transaction_acceptance(node, p2p, tx, with_witness, accepted, reason=No - use the getrawmempool rpc to check for acceptance.""" reason = [reason] if reason else [] with node.assert_debug_log(expected_msgs=reason): - p2p.send_message(msg_witness_tx(tx) if with_witness else msg_tx(tx)) + p2p.send_message(msg_tx(tx) if with_witness else msg_no_witness_tx(tx)) p2p.sync_with_ping() assert_equal(tx.hash in node.getrawmempool(), accepted) + def test_witness_block(node, p2p, block, accepted, with_witness=True, reason=None): """Send a block to the node and check that it's accepted @@ -191,6 +192,7 @@ class SegWitTest(BitcoinTestFramework): ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0", "-segwitheight={}".format(SEGWIT_HEIGHT)], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=1", "-segwitheight=-1"] ] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -310,9 +312,9 @@ class SegWitTest(BitcoinTestFramework): # Check that serializing it with or without witness is the same # This is a sanity check of our testing framework. - assert_equal(msg_tx(tx).serialize(), msg_witness_tx(tx).serialize()) + assert_equal(msg_no_witness_tx(tx).serialize(), msg_tx(tx).serialize()) - self.test_node.send_message(msg_witness_tx(tx)) + self.test_node.send_message(msg_tx(tx)) self.test_node.sync_with_ping() # make sure the tx was processed assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later @@ -1116,7 +1118,7 @@ class SegWitTest(BitcoinTestFramework): MAX_PROGRAM_LENGTH = 10000 # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. - long_witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 63 + [OP_TRUE]) + long_witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 63 + [OP_TRUE]) assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 long_witness_hash = sha256(long_witness_program) long_script_pubkey = CScript([OP_0, long_witness_hash]) @@ -1140,7 +1142,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Try again with one less byte in the witness program - witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 62 + [OP_TRUE]) + witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) assert len(witness_program) == MAX_PROGRAM_LENGTH witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, witness_hash]) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 8979251a26..664a15a5ec 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -15,6 +15,7 @@ class RPCBindTest(BitcoinTestFramework): self.setup_clean_chain = True self.bind_to_localhost_only = False self.num_nodes = 1 + self.supports_cli = False def setup_network(self): self.add_nodes(self.num_nodes, None) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 278ce6d911..adf6f1ca4f 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -49,6 +49,7 @@ class BlockchainTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + self.supports_cli = False def run_test(self): self.mine_chain() diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 2a64a29967..aa7f12848c 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -22,6 +22,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py index 42128d5767..42d7d59d56 100755 --- a/test/functional/rpc_deriveaddresses.py +++ b/test/functional/rpc_deriveaddresses.py @@ -10,7 +10,6 @@ from test_framework.util import assert_equal, assert_raises_rpc_error class DeriveaddressesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.supports_cli = 1 def run_test(self): assert_raises_rpc_error(-5, "Missing checksum", self.nodes[0].deriveaddresses, "a") diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 index 0000000000..7527bdfb08 --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / 'regtest' / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index c956af1cbe..6f1ae0d3ba 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -28,6 +28,9 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True + # This test isn't testing tx relay. Set whitelist on the peers for + # instant tx relay. + self.extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -41,6 +44,7 @@ class RawTransactionsTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 3) def run_test(self): + self.log.info("Connect nodes, set fees, generate blocks, and sync") self.min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate @@ -88,9 +92,11 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_option_feerate() self.test_address_reuse() self.test_option_subtract_fee_from_outputs() + self.test_subtract_fee_with_presets() def test_change_position(self): - # ensure that setting changePosition in fundraw with an exact match is handled properly + """Ensure setting changePosition in fundraw with an exact match is handled properly.""" + self.log.info("Test fundrawtxn changePosition option") rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) assert_equal(rawmatch["changepos"], -1) @@ -115,9 +121,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() def test_simple(self): - ############### - # simple test # - ############### + self.log.info("Test fundrawtxn") inputs = [ ] outputs = { self.nodes[0].getnewaddress() : 1.0 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) @@ -127,9 +131,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert len(dec_tx['vin']) > 0 #test that we have enough inputs def test_simple_two_coins(self): - ############################## - # simple test with two coins # - ############################## + self.log.info("Test fundrawtxn with 2 coins") inputs = [ ] outputs = { self.nodes[0].getnewaddress() : 2.2 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) @@ -141,9 +143,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') def test_simple_two_outputs(self): - ################################ - # simple test with two outputs # - ################################ + self.log.info("Test fundrawtxn with 2 outputs") + inputs = [ ] outputs = { self.nodes[0].getnewaddress() : 2.6, self.nodes[1].getnewaddress() : 2.5 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) @@ -159,9 +160,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') def test_change(self): - ######################################################################### - # test a fundrawtransaction with a VIN greater than the required amount # - ######################################################################### + self.log.info("Test fundrawtxn with a vin > required amount") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] @@ -181,9 +180,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(fee + totalOut, utx['amount']) #compare vin total and totalout+fee def test_no_change(self): - ##################################################################### - # test a fundrawtransaction with which will not get a change output # - ##################################################################### + self.log.info("Test fundrawtxn not having a change output") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] @@ -203,9 +200,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(fee + totalOut, utx['amount']) #compare vin total and totalout+fee def test_invalid_option(self): - #################################################### - # test a fundrawtransaction with an invalid option # - #################################################### + self.log.info("Test fundrawtxn with an invalid option") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -220,9 +215,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Unexpected key reserveChangeKey", lambda: self.nodes[2].fundrawtransaction(hexstring=rawtx, options={'reserveChangeKey': True})) def test_invalid_change_address(self): - ############################################################ - # test a fundrawtransaction with an invalid change address # - ############################################################ + self.log.info("Test fundrawtxn with an invalid change address") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -234,9 +227,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "changeAddress must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) def test_valid_change_address(self): - ############################################################ - # test a fundrawtransaction with a provided change address # - ############################################################ + self.log.info("Test fundrawtxn with a provided change address") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -253,9 +244,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(change, out['scriptPubKey']['addresses'][0]) def test_change_type(self): - ######################################################### - # test a fundrawtransaction with a provided change type # - ######################################################### + self.log.info("Test fundrawtxn with a provided change type") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -268,9 +257,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal('witness_v0_keyhash', dec_tx['vout'][rawtx['changepos']]['scriptPubKey']['type']) def test_coin_selection(self): - ######################################################################### - # test a fundrawtransaction with a VIN smaller than the required amount # - ######################################################################### + self.log.info("Test fundrawtxn with a vin < required amount") utx = get_unspent(self.nodes[2].listunspent(), 1) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] @@ -302,9 +289,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(dec_tx['vout']), 2) def test_two_vin(self): - ########################################### - # test a fundrawtransaction with two VINs # - ########################################### + self.log.info("Test fundrawtxn with 2 vins") utx = get_unspent(self.nodes[2].listunspent(), 1) utx2 = get_unspent(self.nodes[2].listunspent(), 5) @@ -335,9 +320,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(matchingIns, 2) #we now must see two vins identical to vins given as params def test_two_vin_two_vout(self): - ######################################################### - # test a fundrawtransaction with two VINs and two vOUTs # - ######################################################### + self.log.info("Test fundrawtxn with 2 vins and 2 vouts") utx = get_unspent(self.nodes[2].listunspent(), 1) utx2 = get_unspent(self.nodes[2].listunspent(), 5) @@ -360,52 +343,54 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(dec_tx['vout']), 3) def test_invalid_input(self): - ############################################## - # test a fundrawtransaction with invalid vin # - ############################################## + self.log.info("Test fundrawtxn with an invalid vin") inputs = [ {'txid' : "1c7f966dab21119bac53213a2bc7532bff1fa844c124fd750a7d0b1332440bd1", 'vout' : 0} ] #invalid vin! outputs = { self.nodes[0].getnewaddress() : 1.0} rawtx = self.nodes[2].createrawtransaction(inputs, outputs) assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) def test_fee_p2pkh(self): - ############################################################ - #compare fee of a standard pubkeyhash transaction + """Compare fee of a standard pubkeyhash transaction.""" + self.log.info("Test fundrawtxn p2pkh fee") inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1.1) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_fee_p2pkh_multi_out(self): - ############################################################ - #compare fee of a standard pubkeyhash transaction with multiple outputs + """Compare fee of a standard pubkeyhash transaction with multiple outputs.""" + self.log.info("Test fundrawtxn p2pkh fee with multiple outputs") inputs = [] - outputs = {self.nodes[1].getnewaddress():1.1,self.nodes[1].getnewaddress():1.2,self.nodes[1].getnewaddress():0.1,self.nodes[1].getnewaddress():1.3,self.nodes[1].getnewaddress():0.2,self.nodes[1].getnewaddress():0.3} + outputs = { + self.nodes[1].getnewaddress():1.1, + self.nodes[1].getnewaddress():1.2, + self.nodes[1].getnewaddress():0.1, + self.nodes[1].getnewaddress():1.3, + self.nodes[1].getnewaddress():0.2, + self.nodes[1].getnewaddress():0.3, + } rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendmany("", outputs) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_fee_p2sh(self): - ############################################################ - #compare fee of a 2of2 multisig p2sh transaction - - # create 2of2 addr + """Compare fee of a 2-of-2 multisig p2sh transaction.""" + # Create 2-of-2 addr. addr1 = self.nodes[1].getnewaddress() addr2 = self.nodes[1].getnewaddress() @@ -419,20 +404,19 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_fee_4of5(self): - ############################################################ - #compare fee of a standard pubkeyhash transaction + """Compare fee of a standard pubkeyhash transaction.""" + self.log.info("Test fundrawtxn fee with 4-of-5 addresses") - # create 4of5 addr + # Create 4-of-5 addr. addr1 = self.nodes[1].getnewaddress() addr2 = self.nodes[1].getnewaddress() addr3 = self.nodes[1].getnewaddress() @@ -445,40 +429,52 @@ class RawTransactionsTest(BitcoinTestFramework): addr4Obj = self.nodes[1].getaddressinfo(addr4) addr5Obj = self.nodes[1].getaddressinfo(addr5) - mSigObj = self.nodes[1].addmultisigaddress(4, [addr1Obj['pubkey'], addr2Obj['pubkey'], addr3Obj['pubkey'], addr4Obj['pubkey'], addr5Obj['pubkey']])['address'] + mSigObj = self.nodes[1].addmultisigaddress( + 4, + [ + addr1Obj['pubkey'], + addr2Obj['pubkey'], + addr3Obj['pubkey'], + addr4Obj['pubkey'], + addr5Obj['pubkey'], + ] + )['address'] inputs = [] outputs = {mSigObj:1.1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_spend_2of2(self): - ############################################################ - # spend a 2of2 multisig transaction over fundraw + """Spend a 2-of-2 multisig transaction over fundraw.""" + self.log.info("Test fundrawtxn spending 2-of-2 multisig") - # create 2of2 addr + # Create 2-of-2 addr. addr1 = self.nodes[2].getnewaddress() addr2 = self.nodes[2].getnewaddress() addr1Obj = self.nodes[2].getaddressinfo(addr1) addr2Obj = self.nodes[2].getaddressinfo(addr2) - mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']])['address'] + mSigObj = self.nodes[2].addmultisigaddress( + 2, + [ + addr1Obj['pubkey'], + addr2Obj['pubkey'], + ] + )['address'] - - # send 1.2 BTC to msig addr + # Send 1.2 BTC to msig addr. self.nodes[0].sendtoaddress(mSigObj, 1.2) - self.sync_all() - self.nodes[1].generate(1) + self.nodes[0].generate(1) self.sync_all() oldBalance = self.nodes[1].getbalance() @@ -489,35 +485,18 @@ class RawTransactionsTest(BitcoinTestFramework): signedTx = self.nodes[2].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[2].sendrawtransaction(signedTx['hex']) - self.sync_all() - self.nodes[1].generate(1) + self.nodes[2].generate(1) self.sync_all() - # make sure funds are received at node1 + # Make sure funds are received at node1. assert_equal(oldBalance+Decimal('1.10000000'), self.nodes[1].getbalance()) def test_locked_wallet(self): - ############################################################ - # locked wallet test - self.nodes[1].encryptwallet("test") - self.stop_nodes() + self.log.info("Test fundrawtxn with locked wallet") - self.start_nodes() - # This test is not meant to test fee estimation and we'd like - # to be sure all txs are sent at a consistent desired feerate - for node in self.nodes: - node.settxfee(self.min_relay_tx_fee) - - connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[0], 2) - connect_nodes(self.nodes[0], 3) - # Again lock the watchonly UTXO or nodes[0] may spend it, because - # lockunspent is memory-only and thus lost on restart - self.nodes[0].lockunspent(False, [{"txid": self.watchonly_txid, "vout": self.watchonly_vout}]) - self.sync_all() + self.nodes[1].encryptwallet("test") - # drain the keypool + # Drain the keypool. self.nodes[1].getnewaddress() self.nodes[1].getrawchangeaddress() inputs = [] @@ -527,7 +506,7 @@ class RawTransactionsTest(BitcoinTestFramework): # creating the key must be impossible because the wallet is locked assert_raises_rpc_error(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) - #refill the keypool + # Refill the keypool. self.nodes[1].walletpassphrase("test", 100) self.nodes[1].keypoolrefill(8) #need to refill the keypool to get an internal change address self.nodes[1].walletlock() @@ -541,25 +520,23 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[1].createrawtransaction(inputs, outputs) fundedTx = self.nodes[1].fundrawtransaction(rawtx) - #now we need to unlock + # Now we need to unlock. self.nodes[1].walletpassphrase("test", 600) signedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(signedTx['hex']) self.nodes[1].generate(1) self.sync_all() - # make sure funds are received at node1 + # Make sure funds are received at node1. assert_equal(oldBalance+Decimal('51.10000000'), self.nodes[0].getbalance()) def test_many_inputs_fee(self): - ############################################### - # multiple (~19) inputs tx test | Compare fee # - ############################################### + """Multiple (~19) inputs tx test | Compare fee.""" + self.log.info("Test fundrawtxn fee with many inputs") - #empty node1, send some small coins from node0 to node1 + # Empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) - self.sync_all() - self.nodes[0].generate(1) + self.nodes[1].generate(1) self.sync_all() for i in range(0,20): @@ -567,29 +544,27 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() - #fund a tx with ~20 small inputs + # Fund a tx with ~20 small inputs. inputs = [] outputs = {self.nodes[0].getnewaddress():0.15,self.nodes[0].getnewaddress():0.04} rawtx = self.nodes[1].createrawtransaction(inputs, outputs) fundedTx = self.nodes[1].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[1].sendmany("", outputs) signedFee = self.nodes[1].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance * 19 #~19 inputs def test_many_inputs_send(self): - ############################################# - # multiple (~19) inputs tx test | sign/send # - ############################################# + """Multiple (~19) inputs tx test | sign/send.""" + self.log.info("Test fundrawtxn sign+send with many inputs") - #again, empty node1, send some small coins from node0 to node1 + # Again, empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) - self.sync_all() - self.nodes[0].generate(1) + self.nodes[1].generate(1) self.sync_all() for i in range(0,20): @@ -597,7 +572,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() - #fund a tx with ~20 small inputs + # Fund a tx with ~20 small inputs. oldBalance = self.nodes[0].getbalance() inputs = [] @@ -606,15 +581,12 @@ class RawTransactionsTest(BitcoinTestFramework): fundedTx = self.nodes[1].fundrawtransaction(rawtx) fundedAndSignedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(fundedAndSignedTx['hex']) - self.sync_all() - self.nodes[0].generate(1) + self.nodes[1].generate(1) self.sync_all() assert_equal(oldBalance+Decimal('50.19000000'), self.nodes[0].getbalance()) #0.19+block reward def test_op_return(self): - ##################################################### - # test fundrawtransaction with OP_RETURN and no vin # - ##################################################### + self.log.info("Test fundrawtxn with OP_RETURN and no vin") rawtx = "0100000000010000000000000000066a047465737400000000" dec_tx = self.nodes[2].decoderawtransaction(rawtx) @@ -629,9 +601,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(dec_tx['vout']), 2) # one change output added def test_watchonly(self): - ################################################## - # test a fundrawtransaction using only watchonly # - ################################################## + self.log.info("Test fundrawtxn using only watchonly") inputs = [] outputs = {self.nodes[2].getnewaddress(): self.watchonly_amount / 2} @@ -646,15 +616,13 @@ class RawTransactionsTest(BitcoinTestFramework): assert_greater_than(result["changepos"], -1) def test_all_watched_funds(self): - ############################################################### - # test fundrawtransaction using the entirety of watched funds # - ############################################################### + self.log.info("Test fundrawtxn using entirety of watched funds") inputs = [] outputs = {self.nodes[2].getnewaddress(): self.watchonly_amount} rawtx = self.nodes[3].createrawtransaction(inputs, outputs) - # Backward compatibility test (2nd param is includeWatching) + # Backward compatibility test (2nd param is includeWatching). result = self.nodes[3].fundrawtransaction(rawtx, True) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 2) @@ -673,11 +641,9 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() def test_option_feerate(self): - ####################### - # Test feeRate option # - ####################### + self.log.info("Test fundrawtxn feeRate option") - # Make sure there is exactly one input so coin selection can't skew the result + # Make sure there is exactly one input so coin selection can't skew the result. assert_equal(len(self.nodes[3].listunspent(1)), 1) inputs = [] @@ -692,9 +658,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_fee_amount(result3['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) def test_address_reuse(self): - ################################ - # Test no address reuse occurs # - ################################ + """Test no address reuse occurs.""" + self.log.info("Test fundrawtxn does not reuse addresses") rawtx = self.nodes[3].createrawtransaction(inputs=[], outputs={self.nodes[3].getnewaddress(): 1}) result3 = self.nodes[3].fundrawtransaction(rawtx) @@ -705,15 +670,13 @@ class RawTransactionsTest(BitcoinTestFramework): changeaddress += out['scriptPubKey']['addresses'][0] assert changeaddress != "" nextaddr = self.nodes[3].getnewaddress() - # Now the change address key should be removed from the keypool + # Now the change address key should be removed from the keypool. assert changeaddress != nextaddr def test_option_subtract_fee_from_outputs(self): - ###################################### - # Test subtractFeeFromOutputs option # - ###################################### + self.log.info("Test fundrawtxn subtractFeeFromOutputs option") - # Make sure there is exactly one input so coin selection can't skew the result + # Make sure there is exactly one input so coin selection can't skew the result. assert_equal(len(self.nodes[3].listunspent(1)), 1) inputs = [] @@ -744,39 +707,52 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[3].createrawtransaction(inputs, outputs) result = [self.nodes[3].fundrawtransaction(rawtx), - # split the fee between outputs 0, 2, and 3, but not output 1 + # Split the fee between outputs 0, 2, and 3, but not output 1. self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0, 2, 3]})] dec_tx = [self.nodes[3].decoderawtransaction(result[0]['hex']), self.nodes[3].decoderawtransaction(result[1]['hex'])] - # Nested list of non-change output amounts for each transaction + # Nested list of non-change output amounts for each transaction. output = [[out['value'] for i, out in enumerate(d['vout']) if i != r['changepos']] for d, r in zip(dec_tx, result)] - # List of differences in output amounts between normal and subtractFee transactions + # List of differences in output amounts between normal and subtractFee transactions. share = [o0 - o1 for o0, o1 in zip(output[0], output[1])] - # output 1 is the same in both transactions + # Output 1 is the same in both transactions. assert_equal(share[1], 0) - # the other 3 outputs are smaller as a result of subtractFeeFromOutputs + # The other 3 outputs are smaller as a result of subtractFeeFromOutputs. assert_greater_than(share[0], 0) assert_greater_than(share[2], 0) assert_greater_than(share[3], 0) - # outputs 2 and 3 take the same share of the fee + # Outputs 2 and 3 take the same share of the fee. assert_equal(share[2], share[3]) - # output 0 takes at least as much share of the fee, and no more than 2 satoshis more, than outputs 2 and 3 + # Output 0 takes at least as much share of the fee, and no more than 2 + # satoshis more, than outputs 2 and 3. assert_greater_than_or_equal(share[0], share[2]) assert_greater_than_or_equal(share[2] + Decimal(2e-8), share[0]) - # the fee is the same in both transactions + # The fee is the same in both transactions. assert_equal(result[0]['fee'], result[1]['fee']) - # the total subtracted from the outputs is equal to the fee + # The total subtracted from the outputs is equal to the fee. assert_equal(share[0] + share[2] + share[3], result[0]['fee']) + def test_subtract_fee_with_presets(self): + self.log.info("Test fundrawtxn subtract fee from outputs with preset inputs that are sufficient") + + addr = self.nodes[0].getnewaddress() + txid = self.nodes[0].sendtoaddress(addr, 10) + vout = find_vout_for_address(self.nodes[0], txid, addr) + + rawtx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], [{self.nodes[0].getnewaddress(): 5}]) + fundedtx = self.nodes[0].fundrawtransaction(rawtx, {'subtractFeeFromOutputs': [0]}) + signedtx = self.nodes[0].signrawtransactionwithwallet(fundedtx['hex']) + self.nodes[0].sendrawtransaction(signedtx['hex']) + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_getaddressinfo_label_deprecation.py b/test/functional/rpc_getaddressinfo_label_deprecation.py new file mode 100755 index 0000000000..5e739ebede --- /dev/null +++ b/test/functional/rpc_getaddressinfo_label_deprecation.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test deprecation of the RPC getaddressinfo `label` field. It has been +superceded by the `labels` field. + +""" +from test_framework.test_framework import BitcoinTestFramework + +class GetAddressInfoLabelDeprecationTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = False + # Start node[0] with -deprecatedrpc=label, and node[1] without. + self.extra_args = [["-deprecatedrpc=label"], []] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_label_with_deprecatedrpc_flag(self): + self.log.info("Test getaddressinfo label with -deprecatedrpc flag") + node = self.nodes[0] + address = node.getnewaddress() + info = node.getaddressinfo(address) + assert "label" in info + + def test_label_without_deprecatedrpc_flag(self): + self.log.info("Test getaddressinfo label without -deprecatedrpc flag") + node = self.nodes[1] + address = node.getnewaddress() + info = node.getaddressinfo(address) + assert "label" not in info + + def run_test(self): + """Test getaddressinfo label with and without -deprecatedrpc flag.""" + self.test_label_with_deprecatedrpc_flag() + self.test_label_without_deprecatedrpc_flag() + + +if __name__ == '__main__': + GetAddressInfoLabelDeprecationTest().main() diff --git a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py new file mode 100755 index 0000000000..3f2e8dee18 --- /dev/null +++ b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test deprecation of RPC getaddressinfo `labels` returning an array +containing a JSON object of `name` and purpose` key-value pairs. It now +returns an array containing only the label name. + +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +LABELS_TO_TEST = frozenset({"" , "New 𝅘𝅥𝅯 $<#>&!рыба Label"}) + +class GetAddressInfoLabelsPurposeDeprecationTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = False + # Start node[0] with -deprecatedrpc=labelspurpose and node[1] without. + self.extra_args = [["-deprecatedrpc=labelspurpose"], []] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_labels(self, node_num, label_name, expected_value): + node = self.nodes[node_num] + address = node.getnewaddress() + if label_name != "": + node.setlabel(address, label_name) + self.log.info(" set label to {}".format(label_name)) + labels = node.getaddressinfo(address)["labels"] + self.log.info(" labels = {}".format(labels)) + assert_equal(labels, expected_value) + + def run_test(self): + """Test getaddressinfo labels with and without -deprecatedrpc flag.""" + self.log.info("Test getaddressinfo labels with -deprecatedrpc flag") + for label in LABELS_TO_TEST: + self.test_labels(node_num=0, label_name=label, expected_value=[{"name": label, "purpose": "receive"}]) + + self.log.info("Test getaddressinfo labels without -deprecatedrpc flag") + for label in LABELS_TO_TEST: + self.test_labels(node_num=1, label_name=label, expected_value=[label]) + + +if __name__ == '__main__': + GetAddressInfoLabelsPurposeDeprecationTest().main() diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index efab69ac26..57794ae973 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -33,6 +33,7 @@ class GetblockstatsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + self.supports_cli = False def get_stats(self): return [self.nodes[0].getblockstats(hash_or_height=self.start_height + i) for i in range(self.max_stat_pos+1)] diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index 78d6e78aed..027ae368e7 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC help output.""" @@ -13,6 +13,7 @@ import os class HelpRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.supports_cli = False def run_test(self): self.test_categories() diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index 595b40f7cb..1fdc134f97 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -5,7 +5,7 @@ """Test the invalidateblock RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR from test_framework.util import ( assert_equal, connect_nodes, @@ -62,7 +62,7 @@ class InvalidateTest(BitcoinTestFramework): wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") - blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-1]) @@ -74,7 +74,7 @@ class InvalidateTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) self.log.info("Verify that we reconsider all descendants") - blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-2]) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 8a3f8c6f06..c8517d719e 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -19,10 +19,18 @@ from test_framework.authproxy import JSONRPCException class RpcMiscTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.supports_cli = False def run_test(self): node = self.nodes[0] + self.log.info("test CHECK_NONFATAL") + assert_raises_rpc_error( + -1, + "Internal bug detected: 'request.params.size() != 100'", + lambda: node.echo(*[0] * 100), + ) + self.log.info("test getmemoryinfo") memory = node.getmemoryinfo()['locked'] assert_greater_than(memory['used'], 0) diff --git a/test/functional/rpc_named_arguments.py b/test/functional/rpc_named_arguments.py index ecac9c2f82..41b9312969 100755 --- a/test/functional/rpc_named_arguments.py +++ b/test/functional/rpc_named_arguments.py @@ -13,6 +13,7 @@ from test_framework.util import ( class NamedArgumentTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.supports_cli = False def run_test(self): node = self.nodes[0] diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 615f9abbe0..376bb35f07 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC calls related to net. @@ -45,6 +45,7 @@ class NetTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [["-minrelaytxfee=0.00001000"],["-minrelaytxfee=0.00000500"]] + self.supports_cli = False def run_test(self): self.log.info('Connect nodes both way') diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 0663ffdf5b..8386e47411 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -36,6 +36,7 @@ class PreciousTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 + self.supports_cli = False def setup_network(self): self.setup_nodes() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 61572654e0..33af819d34 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -32,6 +32,7 @@ class PSBTTest(BitcoinTestFramework): ["-walletrbf=0"], [] ] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -416,5 +417,25 @@ class PSBTTest(BitcoinTestFramework): analyzed = self.nodes[0].analyzepsbt(signed) assert analyzed['inputs'][0]['has_utxo'] and analyzed['inputs'][0]['is_final'] and analyzed['next'] == 'extractor' + self.log.info("PSBT spending unspendable outputs should have error message and Creator as next") + analysis = self.nodes[0].analyzepsbt('cHNidP8BAJoCAAAAAljoeiG1ba8MI76OcHBFbDNvfLqlyHV5JPVFiHuyq911AAAAAAD/////g40EJ9DsZQpoqka7CwmK6kQiwHGyyng1Kgd5WdB86h0BAAAAAP////8CcKrwCAAAAAAWAEHYXCtx0AYLCcmIauuBXlCZHdoSTQDh9QUAAAAAFv8/wADXYP/7//////8JxOh0LR2HAI8AAAAAAAEBIADC6wsAAAAAF2oUt/X69ELjeX2nTof+fZ10l+OyAokDAQcJAwEHEAABAACAAAEBIADC6wsAAAAAF2oUt/X69ELjeX2nTof+fZ10l+OyAokDAQcJAwEHENkMak8AAAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 spends unspendable output') + + self.log.info("PSBT with invalid values should have error message and Creator as next") + analysis = self.nodes[0].analyzepsbt('cHNidP8BAHECAAAAAfA00BFgAm6tp86RowwH6BMImQNL5zXUcTT97XoLGz0BAAAAAAD/////AgD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XL87QKVAAAAABYAFPck4gF7iL4NL4wtfRAKgQbghiTUAAAAAAABAR8AgIFq49AHABYAFJUDtxf2PHo641HEOBOAIvFMNTr2AAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 has invalid value') + + analysis = self.nodes[0].analyzepsbt('cHNidP8BAHECAAAAAfA00BFgAm6tp86RowwH6BMImQNL5zXUcTT97XoLGz0BAAAAAAD/////AgCAgWrj0AcAFgAUKNw0x8HRctAgmvoevm4u1SbN7XL87QKVAAAAABYAFPck4gF7iL4NL4wtfRAKgQbghiTUAAAAAAABAR8A8gUqAQAAABYAFJUDtxf2PHo641HEOBOAIvFMNTr2AAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Output amount invalid') + + analysis = self.nodes[0].analyzepsbt('cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 specifies invalid prevout') + + assert_raises_rpc_error(-25, 'Missing inputs', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index 74fea07350..14cad3d1b8 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -53,6 +53,7 @@ class RawTransactionsTest(BitcoinTestFramework): ["-txindex"], ["-txindex"], ] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -209,7 +210,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[2].signrawtransactionwithwallet(rawtx) # This will raise an exception since there are missing inputs - assert_raises_rpc_error(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) + assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx['hex']) ##################################### # getrawtransaction with block hash # diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 9f94d11a93..c3d34be0dd 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -54,7 +54,7 @@ class ScantxoutsetTest(BitcoinTestFramework): self.log.info("Stop node, remove wallet, mine again some blocks...") self.stop_node(0) - shutil.rmtree(os.path.join(self.nodes[0].datadir, "regtest", 'wallets')) + shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')) self.start_node(0) self.nodes[0].generate(110) @@ -116,5 +116,12 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8"]) assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)#vchwd07g', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)#z2t3ypsa']) + # Check that status and abort don't need second arg + assert_equal(self.nodes[0].scantxoutset("status"), None) + assert_equal(self.nodes[0].scantxoutset("abort"), False) + + # Check that second arg is needed for start + assert_raises_rpc_error(-1, "scanobjects argument is required for the start action", self.nodes[0].scantxoutset, "start") + if __name__ == '__main__': ScantxoutsetTest().main() diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index b1d2b6f431..1cc1fb164b 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -26,7 +26,7 @@ class SetBanTests(BitcoinTestFramework): self.nodes[1].setban("127.0.0.1", "add") # Node 0 should not be able to reconnect - with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'], timeout=5): + with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'], timeout=50): self.restart_node(1, []) self.nodes[0].addnode("127.0.0.1:" + str(p2p_port(1)), "onetry") diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index 8913b8698d..ca8be42d3b 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 8bbb3c04fa..b75ce15f2e 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiple RPC users.""" @@ -35,6 +35,7 @@ def call_with_auth(node, user, password): class HTTPBasicsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.supports_cli = False def setup_chain(self): super().setup_chain() diff --git a/test/functional/rpc_whitelist.py b/test/functional/rpc_whitelist.py new file mode 100755 index 0000000000..219132410b --- /dev/null +++ b/test/functional/rpc_whitelist.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +A test for RPC users with restricted permissions +""" +from test_framework.test_framework import BitcoinTestFramework +import os +from test_framework.util import ( + get_datadir_path, + assert_equal, + str_to_b64str +) +import http.client +import urllib.parse + +def rpccall(node, user, method): + url = urllib.parse.urlparse(node.url) + headers = {"Authorization": "Basic " + str_to_b64str('{}:{}'.format(user[0], user[3]))} + conn = http.client.HTTPConnection(url.hostname, url.port) + conn.connect() + conn.request('POST', '/', '{"method": "' + method + '"}', headers) + resp = conn.getresponse() + conn.close() + return resp + + +class RPCWhitelistTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def setup_chain(self): + super().setup_chain() + # 0 => Username + # 1 => Password (Hashed) + # 2 => Permissions + # 3 => Password Plaintext + self.users = [ + ["user1", "50358aa884c841648e0700b073c32b2e$b73e95fff0748cc0b517859d2ca47d9bac1aa78231f3e48fa9222b612bd2083e", "getbestblockhash,getblockcount,", "12345"], + ["user2", "8650ba41296f62092377a38547f361de$4620db7ba063ef4e2f7249853e9f3c5c3592a9619a759e3e6f1c63f2e22f1d21", "getblockcount", "54321"] + ] + # For exceptions + self.strange_users = [ + # Test empty + ["strangedude", "62d67dffec03836edd698314f1b2be62$c2fb4be29bb0e3646298661123cf2d8629640979cabc268ef05ea613ab54068d", ":", "s7R4nG3R7H1nGZ"], + ["strangedude2", "575c012c7fe4b1e83b9d809412da3ef7$09f448d0acfc19924dd62ecb96004d3c2d4b91f471030dfe43c6ea64a8f658c1", "", "s7R4nG3R7H1nGZ"], + # Test trailing comma + ["strangedude3", "23189c561b5975a56f4cf94030495d61$3a2f6aac26351e2257428550a553c4c1979594e36675bbd3db692442387728c0", ":getblockcount,", "s7R4nG3R7H1nGZ"], + # Test overwrite + ["strangedude4", "990c895760a70df83949e8278665e19a$8f0906f20431ff24cb9e7f5b5041e4943bdf2a5c02a19ef4960dcf45e72cde1c", ":getblockcount, getbestblockhash", "s7R4nG3R7H1nGZ"], + ["strangedude4", "990c895760a70df83949e8278665e19a$8f0906f20431ff24cb9e7f5b5041e4943bdf2a5c02a19ef4960dcf45e72cde1c", ":getblockcount", "s7R4nG3R7H1nGZ"], + # Testing the same permission twice + ["strangedude5", "d12c6e962d47a454f962eb41225e6ec8$2dd39635b155536d3c1a2e95d05feff87d5ba55f2d5ff975e6e997a836b717c9", ":getblockcount,getblockcount", "s7R4nG3R7H1nGZ"] + ] + # These commands shouldn't be allowed for any user to test failures + self.never_allowed = ["getnetworkinfo"] + with open(os.path.join(get_datadir_path(self.options.tmpdir, 0), "bitcoin.conf"), 'a', encoding='utf8') as f: + f.write("\nrpcwhitelistdefault=0\n") + for user in self.users: + f.write("rpcauth=" + user[0] + ":" + user[1] + "\n") + f.write("rpcwhitelist=" + user[0] + ":" + user[2] + "\n") + # Special cases + for strangedude in self.strange_users: + f.write("rpcauth=" + strangedude[0] + ":" + strangedude[1] + "\n") + f.write("rpcwhitelist=" + strangedude[0] + strangedude[2] + "\n") + + + def run_test(self): + for user in self.users: + permissions = user[2].replace(" ", "").split(",") + # Pop all empty items + i = 0 + while i < len(permissions): + if permissions[i] == '': + permissions.pop(i) + + i += 1 + for permission in permissions: + self.log.info("[" + user[0] + "]: Testing a permitted permission (" + permission + ")") + assert_equal(200, rpccall(self.nodes[0], user, permission).status) + for permission in self.never_allowed: + self.log.info("[" + user[0] + "]: Testing a non permitted permission (" + permission + ")") + assert_equal(403, rpccall(self.nodes[0], user, permission).status) + # Now test the strange users + for permission in self.never_allowed: + self.log.info("Strange test 1") + assert_equal(403, rpccall(self.nodes[0], self.strange_users[0], permission).status) + for permission in self.never_allowed: + self.log.info("Strange test 2") + assert_equal(403, rpccall(self.nodes[0], self.strange_users[1], permission).status) + self.log.info("Strange test 3") + assert_equal(200, rpccall(self.nodes[0], self.strange_users[2], "getblockcount").status) + self.log.info("Strange test 4") + assert_equal(403, rpccall(self.nodes[0], self.strange_users[3], "getbestblockhash").status) + self.log.info("Strange test 5") + assert_equal(200, rpccall(self.nodes[0], self.strange_users[4], "getblockcount").status) + +if __name__ == "__main__": + RPCWhitelistTest().main() diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md new file mode 100644 index 0000000000..f6ea9ef682 --- /dev/null +++ b/test/functional/test-shell.md @@ -0,0 +1,186 @@ +Test Shell for Interactive Environments +========================================= + +This document describes how to use the `TestShell` submodule in the functional +test suite. + +The `TestShell` submodule extends the `BitcoinTestFramework` functionality to +external interactive environments for prototyping and educational purposes. Just +like `BitcoinTestFramework`, the `TestShell` allows the user to: + +* Manage regtest bitcoind subprocesses. +* Access RPC interfaces of the underlying bitcoind instances. +* Log events to the functional test logging utility. + +The `TestShell` can be useful in interactive environments where it is necessary +to extend the object lifetime of the underlying `BitcoinTestFramework` between +user inputs. Such environments include the Python3 command line interpreter or +[Jupyter](https://jupyter.org/) notebooks running a Python3 kernel. + +## 1. Requirements + +* Python3 +* `bitcoind` built in the same repository as the `TestShell`. + +## 2. Importing `TestShell` from the Bitcoin Core repository + +We can import the `TestShell` by adding the path of the Bitcoin Core +`test_framework` module to the beginning of the PATH variable, and then +importing the `TestShell` class from the `test_shell` sub-package. + +``` +>>> import sys +>>> sys.path.insert(0, "/path/to/bitcoin/test/functional") +>>> from test_framework.test_shell import TestShell +``` + +The following `TestShell` methods manage the lifetime of the underlying bitcoind +processes and logging utilities. + +* `TestShell.setup()` +* `TestShell.shutdown()` + +The `TestShell` inherits all `BitcoinTestFramework` members and methods, such +as: +* `TestShell.nodes[index].rpc_method()` +* `TestShell.log.info("Custom log message")` + +The following sections demonstrate how to initialize, run, and shut down a +`TestShell` object. + +## 3. Initializing a `TestShell` object + +``` +>>> test = TestShell().setup(num_nodes=2, setup_clean_chain=True) +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Initializing test directory /path/to/bitcoin_func_test_XXXXXXX +``` +The `TestShell` forwards all functional test parameters of the parent +`BitcoinTestFramework` object. The full set of argument keywords which can be +used to initialize the `TestShell` can be found in [section +#6](#custom-testshell-parameters) of this document. + +**Note: Running multiple instances of `TestShell` is not allowed.** Running a +single process also ensures that logging remains consolidated in the same +temporary folder. If you need more bitcoind nodes than set by default (1), +simply increase the `num_nodes` parameter during setup. + +``` +>>> test2 = TestShell().setup() +TestShell is already running! +``` + +## 4. Interacting with the `TestShell` + +Unlike the `BitcoinTestFramework` class, the `TestShell` keeps the underlying +Bitcoind subprocesses (nodes) and logging utilities running until the user +explicitly shuts down the `TestShell` object. + +During the time between the `setup` and `shutdown` calls, all `bitcoind` node +processes and `BitcoinTestFramework` convenience methods can be accessed +interactively. + +**Example: Mining a regtest chain** + +By default, the `TestShell` nodes are initialized with a clean chain. This means +that each node of the `TestShell` is initialized with a block height of 0. + +``` +>>> test.nodes[0].getblockchaininfo()["blocks"] +0 +``` + +We now let the first node generate 101 regtest blocks, and direct the coinbase +rewards to a wallet address owned by the mining node. + +``` +>>> address = test.nodes[0].getnewaddress() +>>> test.nodes[0].generatetoaddress(101, address) +['2b98dd0044aae6f1cca7f88a0acf366a4bfe053c7f7b00da3c0d115f03d67efb', ... +``` +Since the two nodes are both initialized by default to establish an outbound +connection to each other during `setup`, the second node's chain will include +the mined blocks as soon as they propagate. + +``` +>>> test.nodes[1].getblockchaininfo()["blocks"] +101 +``` +The block rewards from the first block are now spendable by the wallet of the +first node. + +``` +>>> test.nodes[0].getbalance() +Decimal('50.00000000') +``` + +We can also log custom events to the logger. + +``` +>>> test.nodes[0].log.info("Successfully mined regtest chain!") +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework.node0 (INFO): Successfully mined regtest chain! +``` + +**Note: Please also consider the functional test +[readme](../test/functional/README.md), which provides an overview of the +test-framework**. Modules such as +[key.py](../test/functional/test_framework/key.py), +[script.py](../test/functional/test_framework/script.py) and +[messages.py](../test/functional/test_framework/messages.py) are particularly +useful in constructing objects which can be passed to the bitcoind nodes managed +by a running `TestShell` object. + +## 5. Shutting the `TestShell` down + +Shutting down the `TestShell` will safely tear down all running bitcoind +instances and remove all temporary data and logging directories. + +``` +>>> test.shutdown() +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Cleaning up /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` +To prevent the logs from being removed after a shutdown, simply set the +`TestShell.options.nocleanup` member to `True`. +``` +>>> test.options.nocleanup = True +>>> test.shutdown() +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Not cleaning up dir /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` + +The following utility consolidates logs from the bitcoind nodes and the +underlying `BitcoinTestFramework`: + +* `/path/to/bitcoin/test/functional/combine_logs.py + '/path/to/bitcoin_func_test_XXXXXXX'` + +## 6. Custom `TestShell` parameters + +The `TestShell` object initializes with the default settings inherited from the +`BitcoinTestFramework` class. The user can override these in +`TestShell.setup(key=value)`. + +**Note:** `TestShell.reset()` will reset test parameters to default values and +can be called after the TestShell is shut down. + +| Test parameter key | Default Value | Description | +|---|---|---| +| `bind_to_localhost_only` | `True` | Binds bitcoind RPC services to `127.0.0.1` if set to `True`.| +| `cachedir` | `"/path/to/bitcoin/test/cache"` | Sets the bitcoind datadir directory. | +| `chain` | `"regtest"` | Sets the chain-type for the underlying test bitcoind processes. | +| `configfile` | `"/path/to/bitcoin/test/config.ini"` | Sets the location of the test framework config file. | +| `coveragedir` | `None` | Records bitcoind RPC test coverage into this directory if set. | +| `loglevel` | `INFO` | Logs events at this level and higher. Can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR` or `CRITICAL`. | +| `nocleanup` | `False` | Cleans up temporary test directory if set to `True` during `shutdown`. | +| `noshutdown` | `False` | Does not stop bitcoind instances after `shutdown` if set to `True`. | +| `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | +| `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | +| `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | +| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | +| `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | +| `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | +| `trace_rpc` | `False` | Logs all RPC calls if set to `True`. | +| `usecli` | `False` | Uses the bitcoin-cli interface for all bitcoind commands instead of directly calling the RPC server. Requires `supports_cli`. | diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 194f2f061b..6a7e91216a 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -12,6 +12,9 @@ from .util import hex_str_to_bytes from . import segwit_addr ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj' +ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97' +# Coins sent to this address can be spent with a witness stack of just OP_TRUE +ADDRESS_BCRT1_P2WSH_OP_TRUE = 'bcrt1qft5p2uhsdcdc3l2ua4ap5qqfg4pjaqlp250x7us7a8qqhrxrxfsqseac85' class AddressType(enum.Enum): diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 25520a2151..4f7a9a8b13 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1105,17 +1105,17 @@ class msg_tx: self.tx.deserialize(f) def serialize(self): - return self.tx.serialize_without_witness() + return self.tx.serialize_with_witness() def __repr__(self): return "msg_tx(tx=%s)" % (repr(self.tx)) -class msg_witness_tx(msg_tx): +class msg_no_witness_tx(msg_tx): __slots__ = () def serialize(self): - return self.tx.serialize_with_witness() + return self.tx.serialize_without_witness() class msg_block: diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index f95c158a68..a9e669fea9 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -478,7 +478,8 @@ class NetworkThread(threading.Thread): wait_until(lambda: not self.network_event_loop.is_running(), timeout=timeout) self.network_event_loop.close() self.join(timeout) - + # Safe to remove event loop. + NetworkThread.network_event_loop = None class P2PDataStore(P2PInterface): """A P2P data store class. diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 780aa5fe03..da92c6325a 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -12,6 +12,7 @@ import os import pdb import random import shutil +import subprocess import sys import tempfile import time @@ -96,15 +97,45 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.nodes = [] self.network_thread = None self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond - self.supports_cli = False + self.supports_cli = True self.bind_to_localhost_only = True self.set_test_params() - - assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + self.parse_args() def main(self): """Main function. This should not be overridden by the subclass test scripts.""" + assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + + try: + self.setup() + self.run_test() + except JSONRPCException: + self.log.exception("JSONRPC error") + self.success = TestStatus.FAILED + except SkipTest as e: + self.log.warning("Test Skipped: %s" % e.message) + self.success = TestStatus.SKIPPED + except AssertionError: + self.log.exception("Assertion failed") + self.success = TestStatus.FAILED + except KeyError: + self.log.exception("Key error") + self.success = TestStatus.FAILED + except subprocess.CalledProcessError as e: + self.log.exception("Called Process failed with '{}'".format(e.output)) + self.success = TestStatus.FAILED + except Exception: + self.log.exception("Unexpected exception caught during testing") + self.success = TestStatus.FAILED + except KeyboardInterrupt: + self.log.warning("Exiting after keyboard interrupt") + self.success = TestStatus.FAILED + finally: + exit_code = self.shutdown() + sys.exit(exit_code) + + def parse_args(self): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") @@ -130,11 +161,16 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): help="use bitcoin-cli instead of RPC for all commands") parser.add_argument("--perf", dest="perf", default=False, action="store_true", help="profile running nodes with perf for the duration of the test") + parser.add_argument("--valgrind", dest="valgrind", default=False, action="store_true", + help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown, valgrind 3.14 or later required") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") self.add_options(parser) self.options = parser.parse_args() + def setup(self): + """Call this method to start up the test framework object with options set.""" + PortSeed.n = self.options.port_seed check_json_precision() @@ -181,33 +217,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.network_thread = NetworkThread() self.network_thread.start() - success = TestStatus.FAILED + if self.options.usecli: + if not self.supports_cli: + raise SkipTest("--usecli specified but test does not support using CLI") + self.skip_if_no_cli() + self.skip_test_if_missing_module() + self.setup_chain() + self.setup_network() - try: - if self.options.usecli: - if not self.supports_cli: - raise SkipTest("--usecli specified but test does not support using CLI") - self.skip_if_no_cli() - self.skip_test_if_missing_module() - self.setup_chain() - self.setup_network() - self.run_test() - success = TestStatus.PASSED - except JSONRPCException: - self.log.exception("JSONRPC error") - except SkipTest as e: - self.log.warning("Test Skipped: %s" % e.message) - success = TestStatus.SKIPPED - except AssertionError: - self.log.exception("Assertion failed") - except KeyError: - self.log.exception("Key error") - except Exception: - self.log.exception("Unexpected exception caught during testing") - except KeyboardInterrupt: - self.log.warning("Exiting after keyboard interrupt") + self.success = TestStatus.PASSED - if success == TestStatus.FAILED and self.options.pdbonfailure: + def shutdown(self): + """Call this method to shut down the test framework object.""" + + if self.success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() @@ -225,7 +248,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): should_clean_up = ( not self.options.nocleanup and not self.options.noshutdown and - success != TestStatus.FAILED and + self.success != TestStatus.FAILED and not self.options.perf ) if should_clean_up: @@ -238,20 +261,33 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False - if success == TestStatus.PASSED: + if self.success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED - elif success == TestStatus.SKIPPED: + elif self.success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) exit_code = TEST_EXIT_FAILED - logging.shutdown() + # Logging.shutdown will not remove stream- and filehandlers, so we must + # do it explicitly. Handlers are removed so the next test run can apply + # different log handler settings. + # See: https://docs.python.org/3/library/logging.html#logging.shutdown + for h in list(self.log.handlers): + h.flush() + h.close() + self.log.removeHandler(h) + rpc_logger = logging.getLogger("BitcoinRPC") + for h in list(rpc_logger.handlers): + h.flush() + rpc_logger.removeHandler(h) if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) - sys.exit(exit_code) + + self.nodes.clear() + return exit_code # Methods to override in subclass test scripts. def set_test_params(self): @@ -364,6 +400,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args=extra_args[i], use_cli=self.options.usecli, start_perf=self.options.perf, + use_valgrind=self.options.valgrind, )) def start_node(self, i, *args, **kwargs): @@ -566,6 +603,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not self.is_wallet_compiled(): raise SkipTest("wallet has not been compiled.") + def skip_if_no_wallet_tool(self): + """Skip the running test if bitcoin-wallet has not been compiled.""" + if not self.is_wallet_tool_compiled(): + raise SkipTest("bitcoin-wallet has not been compiled") + def skip_if_no_cli(self): """Skip the running test if bitcoin-cli has not been compiled.""" if not self.is_cli_compiled(): @@ -579,6 +621,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): """Checks whether the wallet module was compiled.""" return self.config["components"].getboolean("ENABLE_WALLET") + def is_wallet_tool_compiled(self): + """Checks whether bitcoin-wallet was compiled.""" + return self.config["components"].getboolean("ENABLE_WALLET_TOOL") + def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" return self.config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d8bfdfd040..0742dbe617 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -30,6 +30,7 @@ from .util import ( rpc_url, wait_until, p2p_port, + EncodeDecimal, ) BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -59,7 +60,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): + def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -96,6 +97,15 @@ class TestNode(): "-debugexclude=leveldb", "-uacomment=testnode%d" % i, ] + if use_valgrind: + default_suppressions_file = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "..", "..", "..", "contrib", "valgrind.supp") + suppressions_file = os.getenv("VALGRIND_SUPPRESSIONS_FILE", + default_suppressions_file) + self.args = ["valgrind", "--suppressions={}".format(suppressions_file), + "--gen-suppressions=all", "--exit-on-first-error=yes", + "--error-exitcode=1", "--quiet"] + self.args self.cli = TestNodeCLI(bitcoin_cli, self.datadir) self.use_cli = use_cli @@ -135,25 +145,6 @@ class TestNode(): assert len(self.PRIV_KEYS) == MAX_NODES return self.PRIV_KEYS[self.index] - def get_mem_rss_kilobytes(self): - """Get the memory usage (RSS) per `ps`. - - Returns None if `ps` is unavailable. - """ - assert self.running - - try: - return int(subprocess.check_output( - ["ps", "h", "-o", "rss", "{}".format(self.process.pid)], - stderr=subprocess.DEVNULL).split()[-1]) - - # Avoid failing on platforms where ps isn't installed. - # - # We could later use something like `psutils` to work across platforms. - except (FileNotFoundError, subprocess.SubprocessError): - self.log.exception("Unable to get memory usage") - return None - def _node_msg(self, msg: str) -> str: """Return a modified msg that identifies this node by its index as a debugging aid.""" return "[node %d] %s" % (self.index, msg) @@ -307,7 +298,9 @@ class TestNode(): wait_until(self.is_node_stopped, timeout=timeout) @contextlib.contextmanager - def assert_debug_log(self, expected_msgs, timeout=2): + def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): + if unexpected_msgs is None: + unexpected_msgs = [] time_end = time.time() + timeout debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: @@ -322,6 +315,9 @@ class TestNode(): dl.seek(prev_size) log = dl.read() print_log = " - " + "\n - ".join(log.splitlines()) + for unexpected_msg in unexpected_msgs: + if re.search(re.escape(unexpected_msg), log, flags=re.MULTILINE): + self._raise_assertion_error('Unexpected message "{}" partially matches log:\n\n{}\n\n'.format(unexpected_msg, print_log)) for expected_msg in expected_msgs: if re.search(re.escape(expected_msg), log, flags=re.MULTILINE) is None: found = False @@ -333,33 +329,6 @@ class TestNode(): self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) @contextlib.contextmanager - def assert_memory_usage_stable(self, *, increase_allowed=0.03): - """Context manager that allows the user to assert that a node's memory usage (RSS) - hasn't increased beyond some threshold percentage. - - Args: - increase_allowed (float): the fractional increase in memory allowed until failure; - e.g. `0.12` for up to 12% increase allowed. - """ - before_memory_usage = self.get_mem_rss_kilobytes() - - yield - - after_memory_usage = self.get_mem_rss_kilobytes() - - if not (before_memory_usage and after_memory_usage): - self.log.warning("Unable to detect memory usage (RSS) - skipping memory check.") - return - - perc_increase_memory_usage = (after_memory_usage / before_memory_usage) - 1 - - if perc_increase_memory_usage > increase_allowed: - self._raise_assertion_error( - "Memory usage increased over threshold of {:.3f}% from {} to {} ({:.3f}%)".format( - increase_allowed * 100, before_memory_usage, after_memory_usage, - perc_increase_memory_usage * 100)) - - @contextlib.contextmanager def profile_with_perf(self, profile_name): """ Context manager that allows easy profiling of node activity using `perf`. @@ -526,7 +495,7 @@ def arg_to_cli(arg): if isinstance(arg, bool): return str(arg).lower() elif isinstance(arg, dict) or isinstance(arg, list): - return json.dumps(arg) + return json.dumps(arg, default=EncodeDecimal) else: return str(arg) diff --git a/test/functional/test_framework/test_shell.py b/test/functional/test_framework/test_shell.py new file mode 100644 index 0000000000..26df128f1f --- /dev/null +++ b/test/functional/test_framework/test_shell.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework + +class TestShell: + """Wrapper Class for BitcoinTestFramework. + + The TestShell class extends the BitcoinTestFramework + rpc & daemon process management functionality to external + python environments. + + It is a singleton class, which ensures that users only + start a single TestShell at a time.""" + + class __TestShell(BitcoinTestFramework): + def set_test_params(self): + pass + + def run_test(self): + pass + + def setup(self, **kwargs): + if self.running: + print("TestShell is already running!") + return + + # Num_nodes parameter must be set + # by BitcoinTestFramework child class. + self.num_nodes = 1 + + # User parameters override default values. + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + elif hasattr(self.options, key): + setattr(self.options, key, value) + else: + raise KeyError(key + " not a valid parameter key!") + + super().setup() + self.running = True + return self + + def shutdown(self): + if not self.running: + print("TestShell is not running!") + else: + super().shutdown() + self.running = False + + def reset(self): + if self.running: + print("Shutdown TestShell before resetting!") + else: + self.num_nodes = None + super().__init__() + + instance = None + + def __new__(cls): + # This implementation enforces singleton pattern, and will return the + # previously initialized instance if available + if not TestShell.instance: + TestShell.instance = TestShell.__TestShell() + TestShell.instance.running = False + return TestShell.instance + + def __getattr__(self, name): + return getattr(self.instance, name) + + def __setattr__(self, name, value): + return setattr(self.instance, name, value) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index cde99a2219..5bb73aee7e 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -190,6 +190,11 @@ def check_json_precision(): if satoshis != 2000000000000003: raise RuntimeError("JSON encode/decode loses precision") +def EncodeDecimal(o): + if isinstance(o, Decimal): + return str(o) + raise TypeError(repr(o) + " is not JSON serializable") + def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) @@ -316,6 +321,7 @@ def initialize_datadir(dirname, n, chain): f.write("listenonion=0\n") f.write("printtoconsole=0\n") f.write("upnp=0\n") + f.write("shrinkdebugfile=0\n") os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index c0dfa4c3f0..eb537015fb 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful util functions for testing the wallet""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index e0b523b718..8b527bcff0 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -136,6 +136,7 @@ BASE_SCRIPTS = [ 'interface_rpc.py', 'rpc_psbt.py', 'rpc_users.py', + 'rpc_whitelist.py', 'feature_proxy.py', 'rpc_signrawtransaction.py', 'wallet_groups.py', @@ -181,6 +182,7 @@ BASE_SCRIPTS = [ 'mining_basic.py', 'wallet_bumpfee.py', 'wallet_bumpfee_totalfee_deprecation.py', + 'wallet_implicitsegwit.py', 'rpc_named_arguments.py', 'wallet_listsinceblock.py', 'p2p_leak.py', @@ -190,6 +192,7 @@ BASE_SCRIPTS = [ 'rpc_uptime.py', 'wallet_resendwallettransactions.py', 'wallet_fallbackfee.py', + 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', 'wallet_create_tx.py', @@ -209,6 +212,8 @@ BASE_SCRIPTS = [ 'p2p_permissions.py', 'feature_blocksdir.py', 'feature_config_args.py', + 'rpc_getaddressinfo_labels_purpose_deprecation.py', + 'rpc_getaddressinfo_label_deprecation.py', 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', @@ -362,9 +367,10 @@ def main(): def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, runs_ci, use_term_control): args = args or [] - # Warn if bitcoind is already running (unix only) + # Warn if bitcoind is already running + # pidof might fail or return an empty string if bitcoind is not running try: - if subprocess.check_output(["pidof", "bitcoind"]) is not None: + if subprocess.check_output(["pidof", "bitcoind"]) not in [b'']: print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) except (OSError, subprocess.SubprocessError): pass diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 355cd7af75..d2629ff1ed 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -23,10 +23,11 @@ class ToolWalletTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_wallet_tool() def bitcoin_wallet_process(self, *args): binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] - args = ['-datadir={}'.format(self.nodes[0].datadir), '-regtest'] + list(args) + args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] + list(args) return subprocess.Popen([binary] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) def assert_raises_tool_error(self, error, *args): @@ -197,7 +198,7 @@ class ToolWalletTest(BitcoinTestFramework): self.log.debug('Wallet file shasum unchanged\n') def run_test(self): - self.wallet_path = os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat') + self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat') self.test_invalid_tool_commands_and_args() # Warning: The following tests are order-dependent. self.test_tool_wallet_info() diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index e86679bc31..1122daaf83 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -18,6 +18,7 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, + wait_until, ) @@ -97,6 +98,7 @@ class AbandonConflictTest(BitcoinTestFramework): # TODO: redo with eviction self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) # Verify txs no longer in either node's mempool assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -124,6 +126,8 @@ class AbandonConflictTest(BitcoinTestFramework): # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.00001"]) + wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) + assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(self.nodes[0].getbalance(), balance) @@ -144,6 +148,7 @@ class AbandonConflictTest(BitcoinTestFramework): # Remove using high relay fee again self.stop_node(0) self.start_node(0, extra_args=["-minrelaytxfee=0.0001"]) + wait_until(lambda: self.nodes[0].getmempoolinfo()['loaded']) assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 479d67fb66..f08606e622 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -83,6 +83,7 @@ class AddressTypeTest(BitcoinTestFramework): # whitelist all peers to speed up tx relay / mempool sync for args in self.extra_args: args.append("-whitelist=127.0.0.1") + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 3c8064ea2d..1ca02a695c 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the avoid_reuse and setwalletflag features.""" @@ -68,6 +68,9 @@ class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 2 + # This test isn't testing txn relay/timing, so set whitelist on the + # peers for instant txn relay. This speeds up the test run time 2-3x. + self.extra_args = [["-whitelist=127.0.0.1"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -83,10 +86,18 @@ class AvoidReuseTest(BitcoinTestFramework): reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) self.test_fund_send_fund_senddirty() reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send() + self.test_fund_send_fund_send("legacy") + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_fund_send_fund_send("p2sh-segwit") + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_fund_send_fund_send("bech32") + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_getbalances_used() def test_persistence(self): '''Test that wallet files persist the avoid_reuse flag.''' + self.log.info("Test wallet files persist avoid_reuse flag") + # Configure node 1 to use avoid_reuse self.nodes[1].setwalletflag('avoid_reuse') @@ -109,6 +120,8 @@ class AvoidReuseTest(BitcoinTestFramework): def test_immutable(self): '''Test immutable wallet flags''' + self.log.info("Test immutable wallet flags") + # Attempt to set the disable_private_keys flag; this should not work assert_raises_rpc_error(-8, "Wallet flag is immutable", self.nodes[1].setwalletflag, 'disable_private_keys') @@ -130,6 +143,7 @@ class AvoidReuseTest(BitcoinTestFramework): the avoid_reuse flag set to false. This means the 10 BTC send should succeed, where it fails in test_fund_send_fund_send. ''' + self.log.info("Test fund send fund send dirty") fundaddr = self.nodes[1].getnewaddress() retaddr = self.nodes[0].getnewaddress() @@ -174,7 +188,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_approx(self.nodes[1].getbalance(), 5, 0.001) assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 5, 0.001) - def test_fund_send_fund_send(self): + def test_fund_send_fund_send(self, second_addr_type): ''' Test the simple case where [1] generates a new address A, then [0] sends 10 BTC to A. @@ -183,8 +197,9 @@ class AvoidReuseTest(BitcoinTestFramework): [1] tries to spend 10 BTC (fails; dirty). [1] tries to spend 4 BTC (succeeds; change address sufficient) ''' + self.log.info("Test fund send fund send") - fundaddr = self.nodes[1].getnewaddress() + fundaddr = self.nodes[1].getnewaddress(label="", address_type="legacy") retaddr = self.nodes[0].getnewaddress() self.nodes[0].sendtoaddress(fundaddr, 10) @@ -205,7 +220,19 @@ class AvoidReuseTest(BitcoinTestFramework): # getbalances should show no used, 5 btc trusted assert_balances(self.nodes[1], mine={"used": 0, "trusted": 5}) - self.nodes[0].sendtoaddress(fundaddr, 10) + # For the second send, we transmute it to a related single-key address + # to make sure it's also detected as re-use + fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] + fund_decoded = self.nodes[0].decodescript(fund_spk) + if second_addr_type == "p2sh-segwit": + new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] + elif second_addr_type == "bech32": + new_fundaddr = fund_decoded["segwit"]["addresses"][0] + else: + new_fundaddr = fundaddr + assert_equal(second_addr_type, "legacy") + + self.nodes[0].sendtoaddress(new_fundaddr, 10) self.nodes[0].generate(1) self.sync_all() @@ -231,5 +258,35 @@ class AvoidReuseTest(BitcoinTestFramework): assert_approx(self.nodes[1].getbalance(), 1, 0.001) assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001) + def test_getbalances_used(self): + ''' + getbalances and listunspent should pick up on reused addresses + immediately, even for address reusing outputs created before the first + transaction was spending from that address + ''' + self.log.info("Test getbalances used category") + + # node under test should be completely empty + assert_equal(self.nodes[1].getbalance(avoid_reuse=False), 0) + + new_addr = self.nodes[1].getnewaddress() + ret_addr = self.nodes[0].getnewaddress() + + # send multiple transactions, reusing one address + for _ in range(11): + self.nodes[0].sendtoaddress(new_addr, 1) + + self.nodes[0].generate(1) + self.sync_all() + + # send transaction that should not use all the available outputs + # per the current coin selection algorithm + self.nodes[1].sendtoaddress(ret_addr, 5) + + # getbalances and listunspent should show the remaining outputs + # in the reused address as used/reused + assert_unspent(self.nodes[1], total_count=2, total_sum=6, reused_count=1, reused_sum=1) + assert_balances(self.nodes[1], mine={"used": 1, "trusted": 5}) + if __name__ == '__main__': AvoidReuseTest().main() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index bb835dc811..16c69f304a 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -107,9 +107,9 @@ class WalletBackupTest(BitcoinTestFramework): self.stop_node(2) def erase_three(self): - os.remove(os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat')) - os.remove(os.path.join(self.nodes[1].datadir, 'regtest', 'wallets', 'wallet.dat')) - os.remove(os.path.join(self.nodes[2].datadir, 'regtest', 'wallets', 'wallet.dat')) + os.remove(os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat')) + os.remove(os.path.join(self.nodes[1].datadir, self.chain, 'wallets', 'wallet.dat')) + os.remove(os.path.join(self.nodes[2].datadir, self.chain, 'wallets', 'wallet.dat')) def run_test(self): self.log.info("Generating initial blockchain") @@ -167,13 +167,13 @@ class WalletBackupTest(BitcoinTestFramework): self.erase_three() # Start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'chainstate')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) # Restore wallets from backup - shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat')) - shutil.copyfile(os.path.join(self.nodes[1].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, 'regtest', 'wallets', 'wallet.dat')) - shutil.copyfile(os.path.join(self.nodes[2].datadir, 'wallet.bak'), os.path.join(self.nodes[2].datadir, 'regtest', 'wallets', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[1].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[2].datadir, 'wallet.bak'), os.path.join(self.nodes[2].datadir, self.chain, 'wallets', 'wallet.dat')) self.log.info("Re-starting nodes") self.start_three() @@ -188,8 +188,8 @@ class WalletBackupTest(BitcoinTestFramework): self.erase_three() #start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'chainstate')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) self.start_three() @@ -209,10 +209,10 @@ class WalletBackupTest(BitcoinTestFramework): # Backup to source wallet file must fail sourcePaths = [ - os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat'), - os.path.join(self.nodes[0].datadir, 'regtest', '.', 'wallets', 'wallet.dat'), - os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', ''), - os.path.join(self.nodes[0].datadir, 'regtest', 'wallets')] + os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat'), + os.path.join(self.nodes[0].datadir, self.chain, '.', 'wallets', 'wallet.dat'), + os.path.join(self.nodes[0].datadir, self.chain, 'wallets', ''), + os.path.join(self.nodes[0].datadir, self.chain, 'wallets')] for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c50dcd987a..a5f9a047ed 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -109,13 +109,51 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + # Before `test_balance()`, we have had two nodes with a balance of 50 + # each and then we: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 60 from node B to node A with fee 0.01 + # + # Then we check the balances: + # + # 1) As is + # 2) With transaction 2 from above with 2x the fee + # + # Prior to #16766, in this situation, the node would immediately report + # a balance of 30 on node B as unconfirmed and trusted. + # + # After #16766, we show that balance as unconfirmed. + # + # The balance is indeed "trusted" and "confirmed" insofar as removing + # the mempool transactions would return at least that much money. But + # the algorithm after #16766 marks it as unconfirmed because the 'taint' + # tracking of transaction trust for summing balances doesn't consider + # which inputs belong to a user. In this case, the change output in + # question could be "destroyed" by replace the 1st transaction above. + # + # The post #16766 behavior is correct; we shouldn't be treating those + # funds as confirmed. If you want to rely on that specific UTXO existing + # which has given you that balance, you cannot, as a third party + # spending the other input would destroy that unconfirmed. + # + # For example, if the test transactions were: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 10 from node B to node A with fee 0.01 + # + # Then our node would report a confirmed balance of 40 + 50 - 10 = 80 + # BTC, which is more than would be available if transaction 1 were + # replaced. + + def test_balances(*, fee_node_1=0): # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1) # change from node 1's send + assert_equal(self.nodes[1].getbalance(), Decimal('0')) # node 1's send had an unsafe input # Same with minconf=0 assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) - assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('0')) # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago # TODO: fix getbalance tracking of coin spentness depth assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) @@ -125,9 +163,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('60')) assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('0')) - assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('30') - fee_node_1) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('30') - fee_node_1) test_balances(fee_node_1=Decimal('0.01')) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 550037923e..15746d312c 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -15,6 +15,7 @@ from test_framework.util import ( connect_nodes, wait_until, ) +from test_framework.wallet_util import test_address class WalletTest(BitcoinTestFramework): @@ -24,6 +25,7 @@ class WalletTest(BitcoinTestFramework): "-acceptnonstdtxn=1", ]] * self.num_nodes self.setup_clean_chain = True + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -390,7 +392,7 @@ class WalletTest(BitcoinTestFramework): for label in [u'рыба', u'𝅘𝅥𝅯']: addr = self.nodes[0].getnewaddress() self.nodes[0].setlabel(addr, label) - assert_equal(self.nodes[0].getaddressinfo(addr)['label'], label) + test_address(self.nodes[0], addr, labels=[label]) assert label in self.nodes[0].listlabels() self.nodes[0].rpc.ensure_ascii = True # restore to default diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 0948d47653..4eb0d19a4f 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -38,7 +38,7 @@ class BumpFeeTest(BitcoinTestFramework): "-walletrbf={}".format(i), "-mintxfee=0.00002", "-deprecatedrpc=totalFee", - "-addresstype=p2sh-segwit", # TODO update constants in test and remove + "-addresstype=bech32", ] for i in range(self.num_nodes)] def skip_test_if_missing_module(self): @@ -78,6 +78,7 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_fails(rbf_node, dest_address) test_dust_to_fee(rbf_node, dest_address) test_settxfee(rbf_node, dest_address) + test_watchonly_psbt(self, peer_node, rbf_node, dest_address) test_rebumping(rbf_node, dest_address) test_rebumping_not_replaceable(rbf_node, dest_address) test_unconfirmed_not_spendable(rbf_node, rbf_node_address) @@ -101,7 +102,9 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): else: bumped_tx = rbf_node.bumpfee(rbfid) assert_equal(bumped_tx["errors"], []) - assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 + assert bumped_tx["fee"] > -rbftx["fee"] + assert_equal(bumped_tx["origfee"], -rbftx["fee"]) + assert "psbt" not in bumped_tx # check that bumped_tx propagates, original tx was evicted and has a wallet conflict self.sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() @@ -211,15 +214,15 @@ def test_small_output_with_feerate_succeeds(rbf_node, dest_address): # Make sure additional inputs exist rbf_node.generatetoaddress(101, rbf_node.getnewaddress()) rbfid = spend_one_input(rbf_node, dest_address) - original_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] - assert_equal(len(original_input_list), 1) - original_txin = original_input_list[0] + input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_equal(len(input_list), 1) + original_txin = input_list[0] # Keep bumping until we out-spend change output tx_fee = 0 while tx_fee < Decimal("0.0005"): - new_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] - new_item = list(new_input_list)[0] - assert_equal(len(original_input_list), 1) + input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + new_item = list(input_list)[0] + assert_equal(len(input_list), 1) assert_equal(original_txin["txid"], new_item["txid"]) assert_equal(original_txin["vout"], new_item["vout"]) rbfid_new_details = rbf_node.bumpfee(rbfid) @@ -246,10 +249,8 @@ def test_dust_to_fee(rbf_node, dest_address): # the bumped tx sets fee=49,900, but it converts to 50,000 rbfid = spend_one_input(rbf_node, dest_address) fulltx = rbf_node.getrawtransaction(rbfid, 1) - # (32-byte p2sh-pwpkh output size + 148 p2pkh spend estimate) * 10k(discard_rate) / 1000 = 1800 - # P2SH outputs are slightly "over-discarding" due to the IsDust calculation assuming it will - # be spent as a P2PKH. - bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 1800}) + # (31-vbyte p2wpkh output size + 67-vbyte p2wpkh spend estimate) * 10k(discard_rate) / 1000 = 980 + bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 980}) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) @@ -272,13 +273,95 @@ def test_settxfee(rbf_node, dest_address): def test_maxtxfee_fails(test, rbf_node, dest_address): - test.restart_node(1, ['-maxtxfee=0.00003'] + test.extra_args[1]) + # size of bumped transaction (p2wpkh, 1 input, 2 outputs): 141 vbytes + # expected bumping feerate of 20 sats/vbyte => 141*20 sats = 0.00002820 btc + test.restart_node(1, ['-maxtxfee=0.000025'] + test.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) rbfid = spend_one_input(rbf_node, dest_address) assert_raises_rpc_error(-4, "Unable to create transaction: Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) test.restart_node(1, test.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) +def test_watchonly_psbt(test, peer_node, rbf_node, dest_address): + priv_rec_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0" + pub_rec_desc = rbf_node.getdescriptorinfo(priv_rec_desc)["descriptor"] + priv_change_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh" + pub_change_desc = rbf_node.getdescriptorinfo(priv_change_desc)["descriptor"] + # Create a wallet with private keys that can sign PSBTs + rbf_node.createwallet(wallet_name="signer", disable_private_keys=False, blank=True) + signer = rbf_node.get_wallet_rpc("signer") + assert signer.getwalletinfo()['private_keys_enabled'] + result = signer.importmulti([{ + "desc": priv_rec_desc, + "timestamp": 0, + "range": [0,1], + "internal": False, + "keypool": False # Keys can only be imported to the keypool when private keys are disabled + }, + { + "desc": priv_change_desc, + "timestamp": 0, + "range": [0, 0], + "internal": True, + "keypool": False + }]) + assert_equal(result, [{'success': True}, {'success': True}]) + + # Create another wallet with just the public keys, which creates PSBTs + rbf_node.createwallet(wallet_name="watcher", disable_private_keys=True, blank=True) + watcher = rbf_node.get_wallet_rpc("watcher") + assert not watcher.getwalletinfo()['private_keys_enabled'] + + result = watcher.importmulti([{ + "desc": pub_rec_desc, + "timestamp": 0, + "range": [0,10], + "internal": False, + "keypool": True, + "watchonly": True + }, + { + "desc": pub_change_desc, + "timestamp": 0, + "range": [0, 10], + "internal": True, + "keypool": True, + "watchonly": True + }]) + assert_equal(result, [{'success': True}, {'success': True}]) + + funding_address1 = watcher.getnewaddress(address_type='bech32') + funding_address2 = watcher.getnewaddress(address_type='bech32') + peer_node.sendmany("", {funding_address1: 0.001, funding_address2: 0.001}) + peer_node.generate(1) + test.sync_all() + + # Create single-input PSBT for transaction to be bumped + psbt = watcher.walletcreatefundedpsbt([], {dest_address:0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] + psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) + psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) + original_txid = watcher.sendrawtransaction(psbt_final["hex"]) + assert_equal(len(watcher.decodepsbt(psbt)["tx"]["vin"]), 1) + + # Bump fee, obnoxiously high to add additional watchonly input + bumped_psbt = watcher.bumpfee(original_txid, {"fee_rate":0.005}) + assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) + assert "txid" not in bumped_psbt + assert_equal(bumped_psbt["origfee"], -watcher.gettransaction(original_txid)["fee"]) + assert not watcher.finalizepsbt(bumped_psbt["psbt"])["complete"] + + # Sign bumped transaction + bumped_psbt_signed = signer.walletprocesspsbt(psbt=bumped_psbt["psbt"], sign=True, sighashtype="ALL", bip32derivs=True) + bumped_psbt_final = watcher.finalizepsbt(bumped_psbt_signed["psbt"]) + assert bumped_psbt_final["complete"] + + # Broadcast bumped transaction + bumped_txid = watcher.sendrawtransaction(bumped_psbt_final["hex"]) + assert bumped_txid in rbf_node.getrawmempool() + assert original_txid not in rbf_node.getrawmempool() + + rbf_node.unloadwallet("watcher") + rbf_node.unloadwallet("signer") def test_rebumping(rbf_node, dest_address): # check that re-bumping the original tx fails, but bumping the bumper succeeds diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index e302e499f4..048b3127ff 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -15,7 +15,6 @@ class CreateWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 1 - self.supports_cli = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 53edf710b9..a39dfc7895 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -137,7 +137,7 @@ class WalletDumpTest(BitcoinTestFramework): # encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') - self.nodes[0].walletpassphrase('test', 10) + self.nodes[0].walletpassphrase('test', 100) # Should be a no-op: self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(wallet_enc_dump) diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index fbcb4e75ba..bc7e3cca59 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Wallet encryption""" diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index d1178611bd..f2fa1d3e40 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet group functionality.""" @@ -16,7 +16,7 @@ class WalletGroupTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [[], [], ['-avoidpartialspends']] - self.rpc_timeout = 120 + self.rpc_timeout = 240 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index fa5d5a8878..7497475b67 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -20,6 +20,7 @@ class WalletHDTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 self.extra_args = [[], ['-keypool=0']] + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -67,11 +68,11 @@ class WalletHDTest(BitcoinTestFramework): self.log.info("Restore backup ...") self.stop_node(1) - # we need to delete the complete regtest directory + # we need to delete the complete chain directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks")) - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallets", "wallet.dat")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) + shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) self.start_node(1) # Assert that derivation is deterministic @@ -92,9 +93,9 @@ class WalletHDTest(BitcoinTestFramework): # Try a RPC based rescan self.stop_node(1) - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks")) - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallets", "wallet.dat")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) + shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) self.start_node(1, extra_args=self.extra_args[1]) connect_nodes(self.nodes[0], 1) self.sync_all() diff --git a/test/functional/wallet_implicitsegwit.py b/test/functional/wallet_implicitsegwit.py new file mode 100755 index 0000000000..a8583e2879 --- /dev/null +++ b/test/functional/wallet_implicitsegwit.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the wallet implicit segwit feature.""" + +import test_framework.address as address +from test_framework.test_framework import BitcoinTestFramework + +# TODO: Might be nice to test p2pk here too +address_types = ('legacy', 'bech32', 'p2sh-segwit') + +def key_to_address(key, address_type): + if address_type == 'legacy': + return address.key_to_p2pkh(key) + elif address_type == 'p2sh-segwit': + return address.key_to_p2sh_p2wpkh(key) + elif address_type == 'bech32': + return address.key_to_p2wpkh(key) + +def send_a_to_b(receive_node, send_node): + keys = {} + for a in address_types: + a_address = receive_node.getnewaddress(address_type=a) + pubkey = receive_node.getaddressinfo(a_address)['pubkey'] + keys[a] = pubkey + for b in address_types: + b_address = key_to_address(pubkey, b) + send_node.sendtoaddress(address=b_address, amount=1) + return keys + +def check_implicit_transactions(implicit_keys, implicit_node): + # The implicit segwit node allows conversion all possible ways + txs = implicit_node.listtransactions(None, 99999) + for a in address_types: + pubkey = implicit_keys[a] + for b in address_types: + b_address = key_to_address(pubkey, b) + assert(('receive', b_address) in tuple((tx['category'], tx['address']) for tx in txs)) + +class ImplicitSegwitTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.supports_cli = False + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.log.info("Manipulating addresses and sending transactions to all variations") + implicit_keys = send_a_to_b(self.nodes[0], self.nodes[1]) + + self.sync_all() + + self.log.info("Checking that transactions show up correctly without a restart") + check_implicit_transactions(implicit_keys, self.nodes[0]) + + self.log.info("Checking that transactions still show up correctly after a restart") + self.restart_node(0) + self.restart_node(1) + + check_implicit_transactions(implicit_keys, self.nodes[0]) + +if __name__ == '__main__': + ImplicitSegwitTest().main() diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 79062a4a29..b8b85b7a19 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -145,6 +145,7 @@ def get_rand_amount(): class ImportRescanTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + len(IMPORT_NODES) + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_import_with_label.py b/test/functional/wallet_import_with_label.py index 2a9051b1e8..6a9d2e8290 100755 --- a/test/functional/wallet_import_with_label.py +++ b/test/functional/wallet_import_with_label.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the behavior of RPC importprivkey on set and unset labels of @@ -36,7 +36,7 @@ class ImportWithLabel(BitcoinTestFramework): address, iswatchonly=True, ismine=False, - label=label) + labels=[label]) self.log.info( "Import the watch-only address's private key without a " @@ -44,10 +44,7 @@ class ImportWithLabel(BitcoinTestFramework): ) priv_key = self.nodes[0].dumpprivkey(address) self.nodes[1].importprivkey(priv_key) - - test_address(self.nodes[1], - address, - label=label) + test_address(self.nodes[1], address, labels=[label]) self.log.info( "Test importaddress without label and importprivkey with label." @@ -59,7 +56,7 @@ class ImportWithLabel(BitcoinTestFramework): address2, iswatchonly=True, ismine=False, - label="") + labels=[""]) self.log.info( "Import the watch-only address's private key with a " @@ -69,9 +66,7 @@ class ImportWithLabel(BitcoinTestFramework): label2 = "Test Label 2" self.nodes[1].importprivkey(priv_key2, label2) - test_address(self.nodes[1], - address2, - label=label2) + test_address(self.nodes[1], address2, labels=[label2]) self.log.info("Test importaddress with label and importprivkey with label.") self.log.info("Import a watch-only address with a label.") @@ -82,7 +77,7 @@ class ImportWithLabel(BitcoinTestFramework): address3, iswatchonly=True, ismine=False, - label=label3_addr) + labels=[label3_addr]) self.log.info( "Import the watch-only address's private key with a " @@ -92,9 +87,7 @@ class ImportWithLabel(BitcoinTestFramework): label3_priv = "Test Label 3 for importprivkey" self.nodes[1].importprivkey(priv_key3, label3_priv) - test_address(self.nodes[1], - address3, - label=label3_priv) + test_address(self.nodes[1], address3, labels=[label3_priv]) self.log.info( "Test importprivkey won't label new dests with the same " @@ -108,7 +101,7 @@ class ImportWithLabel(BitcoinTestFramework): address4, iswatchonly=True, ismine=False, - label=label4_addr, + labels=[label4_addr], embedded=None) self.log.info( @@ -121,12 +114,9 @@ class ImportWithLabel(BitcoinTestFramework): self.nodes[1].importprivkey(priv_key4) embedded_addr = self.nodes[1].getaddressinfo(address4)['embedded']['address'] - test_address(self.nodes[1], - embedded_addr, - label="") - test_address(self.nodes[1], - address4, - label=label4_addr) + test_address(self.nodes[1], embedded_addr, labels=[""]) + + test_address(self.nodes[1], address4, labels=[label4_addr]) self.stop_nodes() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 23748e5dd7..f152fcd1a4 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -121,7 +121,7 @@ class ImportMultiTest(BitcoinTestFramework): self.test_importmulti({"scriptPubKey": key.p2pkh_script, "timestamp": "now", "internal": True, - "label": "Example label"}, + "label": "Unsuccessful labelling for internal addresses"}, success=False, error_code=-8, error_message='Internal addresses should not have a label') @@ -550,7 +550,7 @@ class ImportMultiTest(BitcoinTestFramework): self.log.info("Should not import a p2sh-p2wpkh address from descriptor without checksum and private key") self.test_importmulti({"desc": "sh(wpkh(" + key.pubkey + "))", "timestamp": "now", - "label": "Descriptor import test", + "label": "Unsuccessful P2SH-P2WPKH descriptor import", "keys": [key.privkey]}, success=False, error_code=-5, @@ -558,17 +558,18 @@ class ImportMultiTest(BitcoinTestFramework): # Test importing of a P2SH-P2WPKH address via descriptor + private key key = get_key(self.nodes[0]) + p2sh_p2wpkh_label = "Successful P2SH-P2WPKH descriptor import" self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") self.test_importmulti({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), "timestamp": "now", - "label": "Descriptor import test", + "label": p2sh_p2wpkh_label, "keys": [key.privkey]}, success=True) test_address(self.nodes[1], key.p2sh_p2wpkh_addr, solvable=True, ismine=True, - label="Descriptor import test") + labels=[p2sh_p2wpkh_label]) # Test ranged descriptor fails if range is not specified xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" @@ -628,17 +629,18 @@ class ImportMultiTest(BitcoinTestFramework): # Test importing of a P2PKH address via descriptor key = get_key(self.nodes[0]) + p2pkh_label = "P2PKH descriptor import" self.log.info("Should import a p2pkh address from descriptor") self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), "timestamp": "now", - "label": "Descriptor import test"}, + "label": p2pkh_label}, True, warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) test_address(self.nodes[1], key.p2pkh_addr, solvable=True, ismine=False, - label="Descriptor import test") + labels=[p2pkh_label]) # Test import fails if both desc and scriptPubKey are provided key = get_key(self.nodes[0]) @@ -675,7 +677,7 @@ class ImportMultiTest(BitcoinTestFramework): # Import pubkeys with key origin info self.log.info("Addresses should have hd keypath and master key id after import with key origin") pub_addr = self.nodes[1].getnewaddress() - pub_addr = self.nodes[1].getnewaddress() + pub_addr = self.nodes[1].getnewaddress(address_type="bech32") info = self.nodes[1].getaddressinfo(pub_addr) pub = info['pubkey'] pub_keypath = info['hdkeypath'] @@ -693,7 +695,7 @@ class ImportMultiTest(BitcoinTestFramework): assert_equal(pub_import_info['hdkeypath'], pub_keypath) # Import privkeys with key origin info - priv_addr = self.nodes[1].getnewaddress() + priv_addr = self.nodes[1].getnewaddress(address_type="bech32") info = self.nodes[1].getaddressinfo(priv_addr) priv = self.nodes[1].dumpprivkey(priv_addr) priv_keypath = info['hdkeypath'] @@ -742,8 +744,8 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].createwallet(wallet_name="noprivkeys", disable_private_keys=True) wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") - addr1 = self.nodes[0].getnewaddress() - addr2 = self.nodes[0].getnewaddress() + addr1 = self.nodes[0].getnewaddress(address_type="bech32") + addr2 = self.nodes[0].getnewaddress(address_type="bech32") pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] result = wrpc.importmulti( @@ -761,15 +763,15 @@ class ImportMultiTest(BitcoinTestFramework): assert result[0]['success'] assert result[1]['success'] assert_equal(wrpc.getwalletinfo()["keypoolsize"], 2) - newaddr1 = wrpc.getnewaddress() + newaddr1 = wrpc.getnewaddress(address_type="bech32") assert_equal(addr1, newaddr1) - newaddr2 = wrpc.getnewaddress() + newaddr2 = wrpc.getnewaddress(address_type="bech32") assert_equal(addr2, newaddr2) # Import some public keys to the internal keypool of a no privkey wallet self.log.info("Adding pubkey to internal keypool of disableprivkey wallet") - addr1 = self.nodes[0].getnewaddress() - addr2 = self.nodes[0].getnewaddress() + addr1 = self.nodes[0].getnewaddress(address_type="bech32") + addr2 = self.nodes[0].getnewaddress(address_type="bech32") pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] result = wrpc.importmulti( @@ -789,15 +791,15 @@ class ImportMultiTest(BitcoinTestFramework): assert result[0]['success'] assert result[1]['success'] assert_equal(wrpc.getwalletinfo()["keypoolsize_hd_internal"], 2) - newaddr1 = wrpc.getrawchangeaddress() + newaddr1 = wrpc.getrawchangeaddress(address_type="bech32") assert_equal(addr1, newaddr1) - newaddr2 = wrpc.getrawchangeaddress() + newaddr2 = wrpc.getrawchangeaddress(address_type="bech32") assert_equal(addr2, newaddr2) # Import a multisig and make sure the keys don't go into the keypool self.log.info('Imported scripts with pubkeys should not have their pubkeys go into the keypool') - addr1 = self.nodes[0].getnewaddress() - addr2 = self.nodes[0].getnewaddress() + addr1 = self.nodes[0].getnewaddress(address_type="bech32") + addr2 = self.nodes[0].getnewaddress(address_type="bech32") pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] result = wrpc.importmulti( diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 2e70a9e0a5..829633a050 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -30,7 +30,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - wallet_path = os.path.join(self.nodes[1].datadir, "regtest", "wallets", "wallet.dat") + wallet_path = os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat") wallet_backup_path = os.path.join(self.nodes[1].datadir, "wallet.bak") self.nodes[0].generate(101) diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index b71dae9f40..337d2e55d9 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test label RPCs. @@ -13,6 +13,8 @@ from collections import defaultdict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.wallet_util import test_address + class WalletLabelsTest(BitcoinTestFramework): def set_test_params(self): @@ -152,14 +154,9 @@ class Label: def verify(self, node): if self.receive_address is not None: assert self.receive_address in self.addresses - for address in self.addresses: - assert_equal( - node.getaddressinfo(address)['labels'][0], - {"name": self.name, - "purpose": self.purpose[address]}) - assert_equal(node.getaddressinfo(address)['label'], self.name) - + test_address(node, address, labels=[self.name]) + assert self.name in node.listlabels() assert_equal( node.getaddressesbylabel(self.name), {address: {"purpose": self.purpose[address]} for address in self.addresses}) diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 5e94068930..b0590b149a 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -11,6 +11,7 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet_util import test_address class ReceivedByTest(BitcoinTestFramework): @@ -19,6 +20,7 @@ class ReceivedByTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + self.skip_if_no_cli() def run_test(self): # Generate block to get out of IBD @@ -126,7 +128,7 @@ class ReceivedByTest(BitcoinTestFramework): # set pre-state label = '' address = self.nodes[1].getnewaddress() - assert_equal(self.nodes[1].getaddressinfo(address)['label'], label) + test_address(self.nodes[1], address, labels=[label]) received_by_label_json = [r for r in self.nodes[1].listreceivedbylabel() if r["label"] == label][0] balance_by_label = self.nodes[1].getreceivedbylabel(label) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 4aeb393255..6f248c9bd3 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -5,6 +5,7 @@ """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import BIP125_SEQUENCE_NUMBER from test_framework.util import ( assert_array_result, assert_equal, @@ -12,6 +13,7 @@ from test_framework.util import ( connect_nodes, ) +from decimal import Decimal class ListSinceBlockTest(BitcoinTestFramework): def set_test_params(self): @@ -33,10 +35,12 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_reorg() self.test_double_spend() self.test_double_send() + self.double_spends_filtered() def test_no_blockhash(self): txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) blockhash, = self.nodes[2].generate(1) + blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() txs = self.nodes[0].listtransactions() @@ -44,6 +48,7 @@ class ListSinceBlockTest(BitcoinTestFramework): "category": "receive", "amount": 1, "blockhash": blockhash, + "blockheight": blockheight, "confirmations": 1, }) assert_equal( @@ -273,7 +278,8 @@ class ListSinceBlockTest(BitcoinTestFramework): self.sync_all() # gettransaction should work for txid1 - self.nodes[0].gettransaction(txid1) + tx1 = self.nodes[0].gettransaction(txid1) + assert_equal(tx1['blockheight'], self.nodes[0].getblockheader(tx1['blockhash'])['height']) # listsinceblock(lastblockhash) should now include txid1 in transactions # as well as in removed @@ -291,5 +297,51 @@ class ListSinceBlockTest(BitcoinTestFramework): if tx['txid'] == txid1: assert_equal(tx['confirmations'], 2) + def double_spends_filtered(self): + ''' + `listsinceblock` was returning conflicted transactions even if they + occurred before the specified cutoff blockhash + ''' + spending_node = self.nodes[2] + dest_address = spending_node.getnewaddress() + + tx_input = dict( + sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in spending_node.listunspent())) + rawtx = spending_node.createrawtransaction( + [tx_input], {dest_address: tx_input["amount"] - Decimal("0.00051000"), + spending_node.getrawchangeaddress(): Decimal("0.00050000")}) + signedtx = spending_node.signrawtransactionwithwallet(rawtx) + orig_tx_id = spending_node.sendrawtransaction(signedtx["hex"]) + original_tx = spending_node.gettransaction(orig_tx_id) + + double_tx = spending_node.bumpfee(orig_tx_id) + + # check that both transactions exist + block_hash = spending_node.listsinceblock( + spending_node.getblockhash(spending_node.getblockcount())) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, True) + assert_equal(double_found, True) + + lastblockhash = spending_node.generate(1)[0] + + # check that neither transaction exists + block_hash = spending_node.listsinceblock(lastblockhash) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, False) + assert_equal(double_found, False) + if __name__ == '__main__': ListSinceBlockTest().main() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 997d6e702c..8c44a070b8 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -40,14 +40,15 @@ class ListTransactionsTest(BitcoinTestFramework): {"txid": txid}, {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0}) # mine a block, confirmations should change: - self.nodes[0].generate(1) + blockhash = self.nodes[0].generate(1)[0] + blockheight = self.nodes[0].getblockheader(blockhash)['height'] self.sync_all() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, - {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1}) + {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid}, - {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1}) + {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) # send-to-self: txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index ce0b7e8782..f2fa41b647 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -24,7 +24,6 @@ class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.supports_cli = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -39,7 +38,7 @@ class MultiWalletTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] - data_dir = lambda *p: os.path.join(node.datadir, 'regtest', *p) + data_dir = lambda *p: os.path.join(node.datadir, self.chain, *p) wallet_dir = lambda *p: data_dir('wallets', *p) wallet = lambda name: node.get_wallet_rpc(name) @@ -187,7 +186,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_equal(w4.getbalance(), 3) batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) - assert_equal(batch[0]["result"]["chain"], "regtest") + assert_equal(batch[0]["result"]["chain"], self.chain) assert_equal(batch[1]["result"]["walletname"], "w1") self.log.info('Check for per-wallet settxfee call') diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index f48018e9fb..497a5dd95e 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -90,7 +90,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): # Node0 wallet file is loaded on longest sync'ed node1 self.stop_node(1) self.nodes[0].backupwallet(os.path.join(self.nodes[0].datadir, 'wallet.bak')) - shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, 'regtest', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, self.chain, 'wallet.dat')) self.start_node(1) tx_after_reorg = self.nodes[1].gettransaction(txid) # Check that normal confirmed tx is confirmed again but with different blockhash diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 60d7205887..99559090ee 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -16,6 +16,7 @@ from test_framework.messages import CTransaction, COIN class TxnMallTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index 40eeb4048c..1891cd9190 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -16,6 +16,7 @@ from test_framework.util import ( class TxnMallTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index be8d7714fb..b0c41b2738 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -16,7 +16,6 @@ class CreateWalletWatchonlyTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 1 - self.supports_cli = True def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index f19cf924d2..6b687060e2 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -12,6 +12,39 @@ import sys import subprocess import logging +# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus +FUZZERS_MISSING_CORPORA = [ + "addr_info_deserialize", + "base_encode_decode", + "block", + "block_file_info_deserialize", + "block_filter_deserialize", + "block_header_and_short_txids_deserialize", + "decode_tx", + "fee_rate_deserialize", + "flat_file_pos_deserialize", + "hex", + "integer", + "key_origin_info_deserialize", + "merkle_block_deserialize", + "out_point_deserialize", + "parse_hd_keypath", + "parse_numbers", + "parse_script", + "parse_univalue", + "partial_merkle_tree_deserialize", + "partially_signed_transaction_deserialize", + "prefilled_transaction_deserialize", + "psbt_input_deserialize", + "psbt_output_deserialize", + "pub_key_deserialize", + "script_deserialize", + "strprintf", + "sub_net_deserialize", + "tx_in", + "tx_in_deserialize", + "tx_out", +] def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -78,7 +111,7 @@ def main(): os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), '-help=1', ], - timeout=1, + timeout=10, check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -100,10 +133,14 @@ def main(): def run_once(*, corpus, test_list, build_dir, export_coverage): for t in test_list: + corpus_path = os.path.join(corpus, t) + if t in FUZZERS_MISSING_CORPORA: + os.makedirs(corpus_path, exist_ok=True) args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), '-runs=1', - os.path.join(corpus, t), + '-detect_leaks=0', + corpus_path, ] logging.debug('Run {} with args {}'.format(t, args)) result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py index a33ab17f3f..0a4cc875d0 100755 --- a/test/lint/check-rpc-mappings.py +++ b/test/lint/check-rpc-mappings.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check RPC argument consistency.""" diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 5603456e62..ff3f784437 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 85e8b841b6..caa7affc63 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -1,10 +1,11 @@ #!/bin/sh -# Copyright (c) 2015 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -DIR="$1" +# Strip trailing / from directory path (in case it was added by autocomplete) +DIR="${1%/}" COMMIT="$2" if [ -z "$COMMIT" ]; then COMMIT=HEAD diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index 5bbcae79eb..1aacc09bcc 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -20,4 +20,15 @@ if [[ ${OUTPUT} != "" ]]; then EXIT_CODE=1 fi +# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it +# is undesirable to crash the whole program. See: src/util/check.h +# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. +OUTPUT=$(git grep -nE 'assert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") +if [[ ${OUTPUT} != "" ]]; then + echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi + exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 8607fc4371..bbd94dd6c7 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -13,24 +13,14 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "index/txindex -> validation -> index/txindex" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/bantablemodel -> qt/clientmodel -> qt/bantablemodel" - "qt/bitcoingui -> qt/utilitydialog -> qt/bitcoingui" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" - "qt/bitcoingui -> qt/walletview -> qt/bitcoingui" - "qt/clientmodel -> qt/peertablemodel -> qt/clientmodel" - "qt/paymentserver -> qt/walletmodel -> qt/paymentserver" "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "qt/walletmodel -> qt/walletmodeltransaction -> qt/walletmodel" "txmempool -> validation -> txmempool" - "wallet/coincontrol -> wallet/wallet -> wallet/coincontrol" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" "policy/fees -> txmempool -> validation -> policy/fees" - "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/guiutil" - "txmempool -> validation -> validationinterface -> txmempool" - "wallet/ismine -> wallet/wallet -> wallet/ismine" ) EXIT_CODE=0 diff --git a/test/lint/lint-filenames.sh b/test/lint/lint-filenames.sh index 6716cac0fe..3f7491cd2b 100755 --- a/test/lint/lint-filenames.sh +++ b/test/lint/lint-filenames.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 9f34d0f4dd..2870432bff 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -17,9 +17,13 @@ FALSE_POSITIVES = [ ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), ("src/util/system.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), ("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), ] diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index cb630c78ad..184c3682c8 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -34,7 +34,7 @@ if ! python3 -m doctest test/lint/lint-format-strings.py; then fi for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue)"); do + for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do MATCHING_FILES+=("${MATCHING_FILE}") done if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index 0d654e796e..2d3beaf582 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index d27e45a23f..bb2bd4e56c 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 9a1aa766f7..35e58c2df6 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -1,16 +1,35 @@ #!/usr/bin/env bash +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C KNOWN_VIOLATIONS=( "src/bitcoin-tx.cpp.*stoul" + "src/bitcoin-tx.cpp.*std::to_string" "src/bitcoin-tx.cpp.*trim_right" "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" + "src/qt/optionsmodel.cpp.*std::to_string" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" + "src/rpc/net.cpp.*std::to_string" + "src/rpc/rawtransaction.cpp.*std::to_string" + "src/rpc/util.cpp.*std::to_string" + "src/test/addrman_tests.cpp.*std::to_string" + "src/test/blockchain_tests.cpp.*std::to_string" "src/test/dbwrapper_tests.cpp:.*snprintf" + "src/test/denialofservice_tests.cpp.*std::to_string" + "src/test/fuzz/parse_numbers.cpp:.*atoi" + "src/test/key_tests.cpp.*std::to_string" + "src/test/net_tests.cpp.*std::to_string" + "src/test/settings_tests.cpp.*std::to_string" + "src/test/timedata_tests.cpp.*std::to_string" + "src/test/util/setup_common.cpp.*std::to_string" + "src/test/util_tests.cpp.*std::to_string" + "src/test/util_threadnames_tests.cpp.*std::to_string" "src/torcontrol.cpp:.*atoi" "src/torcontrol.cpp:.*strtol" "src/util/strencodings.cpp:.*atoi" @@ -18,6 +37,7 @@ KNOWN_VIOLATIONS=( "src/util/strencodings.cpp:.*strtoul" "src/util/strencodings.h:.*atoi" "src/util/system.cpp:.*atoi" + "src/wallet/scriptpubkeyman.cpp.*std::to_string" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" @@ -92,6 +112,7 @@ LOCALE_DEPENDENT_FUNCTIONS=( snprintf sprintf sscanf + std::to_string stod stof stoi diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh index 632ed7c812..2fbb4a38e7 100755 --- a/test/lint/lint-logs.sh +++ b/test/lint/lint-logs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -15,6 +15,7 @@ export LC_ALL=C UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ grep -v '\\n"' | \ + grep -v '\.\.\.' | \ grep -v "/\* Continued \*/" | \ grep -v "LogPrint()" | \ grep -v "LogPrintf()") diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 3c82ec19e3..86ac5a930f 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -82,10 +82,10 @@ enabled=( ) if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed. Install by running \"pip3 install flake8\"" + echo "Skipping Python linting since flake8 is not installed." exit 0 elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8 by running \"pip3 install flake8\"" + echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." exit 0 fi diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh index fda22592d3..a666fdfecf 100755 --- a/test/lint/lint-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + # Assert expected shebang lines export LC_ALL=C diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 69fc3cf368..63624e3ae0 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -16,16 +16,45 @@ if [ "$TRAVIS" = "true" ]; then unset LC_ALL fi -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit 0 -fi - # Disabled warnings: disabled=( SC2046 # Quote this to prevent word splitting. SC2086 # Double quote to prevent globbing and word splitting. SC2162 # read without -r will mangle backslashes. ) -shellcheck -e "$(IFS=","; echo "${disabled[*]}")" \ - $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/') +disabled_gitian=( + SC2094 # Make sure not to read and write the same file in the same pipeline. + SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects. + SC2230 # which is non-standard. Use builtin 'command -v' instead. +) + +EXIT_CODE=0 + +if ! command -v shellcheck > /dev/null; then + echo "Skipping shell linting since shellcheck is not installed." + exit $EXIT_CODE +fi + +EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" +if ! shellcheck "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then + EXIT_CODE=1 +fi + +if ! command -v yq > /dev/null; then + echo "Skipping Gitian desriptor scripts checking since yq is not installed." + exit $EXIT_CODE +fi + +EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" +for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') +do + echo + echo "$descriptor" + # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. + SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor") + if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then + EXIT_CODE=1 + fi +done + +exit $EXIT_CODE diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/lint-spelling.ignore-words.txt index b08837c1d4..576ae94098 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/lint-spelling.ignore-words.txt @@ -1,9 +1,7 @@ -cas hights mor mut objext -unselect useable wit unparseable @@ -13,3 +11,4 @@ errorstring keyserver homogenous setban +hist diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index e70b73e1cc..c7a3d0de44 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi diff --git a/test/sanitizer_suppressions/lsan b/test/sanitizer_suppressions/lsan index 90a92a5115..d2cb618d4e 100644 --- a/test/sanitizer_suppressions/lsan +++ b/test/sanitizer_suppressions/lsan @@ -1,5 +1,4 @@ # Suppress warnings triggered in dependencies -leak:libcrypto leak:libqminimal leak:libQt5Core leak:libQt5Gui diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 70eea34363..b9c5c038d0 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -7,6 +7,14 @@ deadlock:WalletBatch # Intentional deadlock in tests deadlock:TestPotentialDeadLockDetected +# Race due to unprotected calls to thread-unsafe BOOST_TEST_MESSAGE from different threads: +# * G_TEST_LOG_FUN in the index thread +# * boost test case invoker (entering a test case) in the main thread +# TODO: get rid of BOOST_ macros, see also https://github.com/bitcoin/bitcoin/issues/8670 +race:blockfilter_index_initial_sync_invoker +race:txindex_initial_sync_invoker +race:validation_block_tests::TestSubscriber + # Wildcard for all gui tests, should be replaced with non-wildcard suppressions race:src/qt/test/* deadlock:src/qt/test/* diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index e7c690fabe..f5de358bcb 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -42,3 +42,45 @@ unsigned-integer-overflow:stl_bvector.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp + +implicit-integer-sign-change:*/include/c++/*/bits/*.h +implicit-integer-sign-change:*/new_allocator.h +implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp +implicit-integer-sign-change:arith_uint256.cpp +implicit-integer-sign-change:bech32.cpp +implicit-integer-sign-change:bloom.cpp +implicit-integer-sign-change:chain.* +implicit-integer-sign-change:coins.h +implicit-integer-sign-change:compat/stdin.cpp +implicit-integer-sign-change:compressor.h +implicit-integer-sign-change:crypto/* +implicit-integer-sign-change:key.cpp +implicit-integer-sign-change:noui.cpp +implicit-integer-sign-change:prevector.h +implicit-integer-sign-change:protocol.cpp +implicit-integer-sign-change:script/bitcoinconsensus.cpp +implicit-integer-sign-change:script/interpreter.cpp +implicit-integer-sign-change:serialize.h +implicit-integer-sign-change:test/arith_uint256_tests.cpp +implicit-integer-sign-change:test/coins_tests.cpp +implicit-integer-sign-change:test/pow_tests.cpp +implicit-integer-sign-change:test/prevector_tests.cpp +implicit-integer-sign-change:test/sighash_tests.cpp +implicit-integer-sign-change:test/streams_tests.cpp +implicit-integer-sign-change:test/transaction_tests.cpp +implicit-integer-sign-change:txmempool.cpp +implicit-integer-sign-change:util/strencodings.* +implicit-integer-sign-change:validation.cpp +implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp +implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h +implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp +implicit-signed-integer-truncation:chain.h +implicit-signed-integer-truncation:crypto/* +implicit-signed-integer-truncation:cuckoocache.h +implicit-signed-integer-truncation:leveldb/* +implicit-signed-integer-truncation:streams.h +implicit-signed-integer-truncation:test/arith_uint256_tests.cpp +implicit-signed-integer-truncation:test/skiplist_tests.cpp +implicit-signed-integer-truncation:torcontrol.cpp +implicit-unsigned-integer-truncation:crypto/* +implicit-unsigned-integer-truncation:leveldb/* |