diff options
394 files changed, 9326 insertions, 6585 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index ed2ab49554..b4850d49c8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,31 +1,57 @@ version: '{branch}.{build}' skip_tags: true -image: Visual Studio 2017 +image: Visual Studio 2019 configuration: Release platform: x64 clone_depth: 5 environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true CLCACHE_SERVER: 1 - PACKAGES: berkeleydb boost-filesystem boost-signals2 boost-test libevent openssl rapidcheck zeromq double-conversion PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.0/Qt5.9.7_ssl_x64_static_vs2017.zip' - QT_DOWNLOAD_HASH: 'D4D35B8112302B67E5610A03421BB3E43FE13F14D9A5F637C22AE60DCEC0E0F5' - QT_LOCAL_PATH: 'C:\Qt5.9.7_ssl_x64_static_vs2017' + QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.4/Qt5.9.8_x64_static_vs2019.zip' + QT_DOWNLOAD_HASH: 'f285cbb02bec3b3f3cc2621e3fa7d5edf0d6a66fa30c57859e583acda954ea80' + QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' + VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' cache: -- C:\tools\vcpkg\installed +- C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt - C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in -- C:\Qt5.9.7_ssl_x64_static_vs2017 +- C:\Qt5.9.8_x64_static_vs2019 install: - cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq -- cmd: echo set(VCPKG_BUILD_TYPE release) >> C:\tools\vcpkg\triplets\%PLATFORM%-windows-static.cmake -- cmd: vcpkg remove --outdated --recurse -- cmd: vcpkg install --triplet %PLATFORM%-windows-static %PACKAGES% > NUL +# Powershell block below is to install the c++ dependencies via vcpkg. The pseudo code is: +# 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file +# will cause the appveyor cache rules to invalidate the directory) +# 2. If the directory is missing: +# a. Update the vcpkg source (including port files) and build the vcpkg binary, +# b. Install the missing packages. +- ps: | + $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt + Write-Host "vcpkg list: $env:PACKAGES" + if(!(Test-Path -Path ($env:VCPKG_INSTALL_PATH))) { + cd c:\tools\vcpkg + $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. + git pull origin master + .\bootstrap-vcpkg.bat + Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" + .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null + cd "$env:APPVEYOR_BUILD_FOLDER" + } + else { + Write-Host "required vcpkg packages already installed." + } + c:\tools\vcpkg\vcpkg integrate install before_build: - ps: clcache -M 536870912 +# Powershell block below is to download and extract the Qt static libraries. The pseudo code is: +# 1. If the Qt destination directory exists assume it is correct and do nothing. To +# force a fresh install of the packages delete the job's appveyor cache. +# 2. Otherwise: +# a. Download the zip file with the prebuilt Qt static libraries. +# b. Check that the downloaded file matches the expected hash. +# c. Extract the zip file to the specific destination path expected by the msbuild projects. - ps: | if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) { Write-Host "Downloading Qt binaries."; @@ -44,17 +70,10 @@ before_build: Write-Host "Qt binaries already present."; } - cmd: python build_msvc\msvc-autogen.py -- ps: $files = (Get-ChildItem -Recurse | where {$_.extension -eq ".vcxproj"}).FullName -- ps: for (${i} = 0; ${i} -lt ${files}.length; ${i}++) { - ${content} = (Get-Content ${files}[${i}]); - ${content} = ${content}.Replace("</RuntimeLibrary>", "</RuntimeLibrary><DebugInformationFormat>None</DebugInformationFormat>"); - ${content} = ${content}.Replace("<WholeProgramOptimization>true", "<WholeProgramOptimization>false"); - Set-Content ${files}[${i}] ${content}; - } - ps: Start-Process clcache-server - ps: fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache) build_script: -- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:n /nologo +- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:q /nologo after_build: - ps: fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance) - ps: clcache -z @@ -64,7 +83,8 @@ test_script: - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - ps: python test\util\bitcoin-util-test.py - cmd: python test\util\rpcauth-test.py -- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast +# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. +- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation artifacts: #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index 9464ec1685..517cd93585 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -45,8 +45,6 @@ task: folder: "/tmp/ccache_dir" depends_built_cache: folder: "/tmp/cirrus-ci-build/depends/built" - depends_sdk_cache: - folder: "/tmp/cirrus-ci-build/depends/sdk-sources" install_script: - apt-get update - apt-get -y install git bash ccache diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md new file mode 100644 index 0000000000..6782218616 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -0,0 +1,20 @@ +--- +name: Good first issue +about: '(Regular devs only): Suggest a new good first issue' +title: '' +labels: good first issue +assignees: '' + +--- + +The purpose of the `good first issue` label is to highlight which issues are suitable for a new contributor without a deep understanding of the codebase. + +Useful skills: + +(For example, “C++11 std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) + +Want to work on this issue? + +You do not need to request permission to start working on this. You are encouraged to comment on the issue if you are planning to work on it. This will help other contributors monitor which issues are actively being addressed and is also an effective way to request assistance if and when you need it. + +For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. diff --git a/.travis.yml b/.travis.yml index 0a14ddf34c..73e62cd2b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ dist: xenial os: linux language: minimal +arch: amd64 cache: ccache: true directories: @@ -46,6 +47,7 @@ stages: env: global: - CI_RETRY_EXE="travis_retry" + - CI_WAIT="while sleep 500; do echo .; done" - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." before_install: - set -o errexit; source ./ci/test/00_setup_env.sh @@ -94,9 +96,11 @@ jobs: - set -o errexit; source ./ci/extended_lint/06_script.sh - stage: test - name: 'ARM [GOAL: install] [unit tests, no functional tests]' + name: 'ARM [GOAL: install] [unit tests, functional tests]' + arch: arm64 env: >- FILE_ENV="./ci/test/00_setup_env_arm.sh" + QEMU_USER_CMD="" # Can run the tests natively without qemu - stage: test name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' @@ -104,40 +108,40 @@ jobs: FILE_ENV="./ci/test/00_setup_env_win64.sh" - stage: test - name: '32-bit + dash [GOAL: install] [GUI: BIP70 enabled]' + name: '32-bit + dash [GOAL: install] [gui]' env: >- FILE_ENV="./ci/test/00_setup_env_i686.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_qt5.sh" + FILE_ENV="./ci/test/00_setup_env_native_qt5.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_trusty.sh" + FILE_ENV="./ci/test/00_setup_env_native_trusty.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no depends, only system libs, sanitizers: thread (TSan), no wallet]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_tsan.sh" + FILE_ENV="./ci/test/00_setup_env_native_tsan.sh" TEST_RUNNER_EXTRA="--exclude feature_block" # Not enough memory on travis machines - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_asan.sh" + FILE_ENV="./ci/test/00_setup_env_native_asan.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address,undefined]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_fuzz.sh" + FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- - FILE_ENV="./ci/test/00_setup_env_amd64_nowallet.sh" + FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" - stage: test name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' @@ -145,11 +149,11 @@ jobs: FILE_ENV="./ci/test/00_setup_env_mac.sh" - stage: test - name: 'macOS 10.14 native [GOAL: install] [GUI: BIP70 enabled] [no depends]' + name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]' os: osx # Use the most recent version: # Xcode 11, macOS 10.14, JDK 12.0.1 # https://docs.travis-ci.com/user/reference/osx/#macos-version osx_image: xcode11 env: >- - FILE_ENV="./ci/test/00_setup_env_mac_functional.sh" + FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/Makefile.am b/Makefile.am index 9e54dc210d..deae2c1b57 100644 --- a/Makefile.am +++ b/Makefile.am @@ -52,7 +52,10 @@ DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ $(top_srcdir)/contrib/bitcoind.bash-completion \ $(top_srcdir)/contrib/debian/copyright \ $(top_srcdir)/contrib/init \ - $(top_srcdir)/contrib/install_db4.sh + $(top_srcdir)/contrib/install_db4.sh \ + $(top_srcdir)/contrib/linearize/linearize-data.py \ + $(top_srcdir)/contrib/linearize/linearize-hashes.py + DIST_SHARE = \ $(top_srcdir)/share/genbuild.sh \ $(top_srcdir)/share/rpcauth @@ -12,7 +12,7 @@ with no central authority: managing transactions and issuing money are carried out collectively by the network. Bitcoin Core is the name of open source software which enables the use of this currency. -For more information, as well as an immediately useable, binary version of +For more information, as well as an immediately usable, binary version of the Bitcoin Core software, see https://bitcoincore.org/en/download/, or read the [original whitepaper](https://bitcoincore.org/bitcoin.pdf). diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index d579dc2ed5..83d054af5f 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,8 +116,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test "x$bitcoin_cv_static_qt" = xyes; then _BITCOIN_QT_FIND_STATIC_PLUGINS AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + if test "x$TARGET_OS" != xandroid; then + _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) + AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + fi if test "x$TARGET_OS" = xwindows; then _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows]) AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) @@ -128,6 +130,9 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AX_CHECK_LINK_FLAG([[-framework IOKit]],[QT_LIBS="$QT_LIBS -framework IOKit"],[AC_MSG_ERROR(could not iokit framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin)],[-lqcocoa]) AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) + elif test "x$TARGET_OS" = xandroid; then + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype -lQt5EglSupport $QT_LIBS" + AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS @@ -345,6 +350,9 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/accessible"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible" fi + if test -d "$qt_plugin_path/platforms/android"; then + QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" + fi if test "x$use_pkgconfig" = xyes; then : dnl m4_ifdef([PKG_CHECK_MODULES],[ diff --git a/build_msvc/README.md b/build_msvc/README.md index 88b1f514bd..59be5298ad 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -12,7 +12,7 @@ Quick Start The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. ``` -vcpkg install --triplet x64-windows-static boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb rapidcheck double-conversion +vcpkg install --triplet x64-windows-static boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent openssl zeromq berkeleydb rapidcheck double-conversion py -3 build_msvc\msvc-autogen.py msbuild /m build_msvc\bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` @@ -40,33 +40,27 @@ The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/d Qt --------------------- -All the Bitcoin Core applications are configured to build with static linking. In order to build the Bitcoin Core Qt applications a static build of Qt is required. +In order to build the Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. -The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. OpenSSL must also be linked into the Qt binaries in order to provide full functionality of the Bitcoin Core Qt programs. An example of the configure command to build Qtv5.9.7 locally to link with Bitcoin Core is shown below (adjust paths accordingly), note it can be expected that the configure and subsequent build will fail numerous times until dependency issues are resolved. +A prebuilt version of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware this download is NOT an officially sanctioned Bitcoin Core distribution and is provided for developer convenience. It should NOT be used for builds that will be used in a production environment or with real funds. -```` -..\Qtv5.9.7_src\configure -developer-build -confirm-license -debug-and-release -opensource -platform win32-msvc -opengl desktop -no-shared -static -no-static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -ltcg -make libs -make tools -no-libjpeg -nomake examples -no-compile-examples -no-dbus -no-libudev -no-qml-debug -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -nomake tests -openssl-linked -IC:\Dev\github\vcpkg\installed\x64-windows-static\include -LC:\Dev\github\vcpkg\installed\x64-windows-static\lib OPENSSL_LIBS="-llibeay32 -lssleay32 -lgdi32 -luser32 -lwsock32 -ladvapi32" -prefix C:\Qt5.9.7_ssl_x64_static_vs2017 -```` - -A prebuilt version for x64 and Visual C++ runtime v141 (Visual Studio 2017) can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware this download is NOT an officially sanctioned Bitcoin Core distribution and is provided for developer convenience. It should NOT be used for builds that will be used in a production environment or with real funds. - -To build Bitcoin Core without Qt unload or disable the bitcoin-qt, libbitcoin_qt and test_bitcoin-qt projects. +To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. Building --------------------- The instructions below use `vcpkg` to install the dependencies. -- Clone `vcpkg` from the [github repository](https://github.com/Microsoft/vcpkg) and install as per the instructions in the main README.md. -- Install the required packages (replace x64 with x86 as required): +- Install [`vcpkg`](https://github.com/Microsoft/vcpkg). +- Install the required packages (replace x64 with x86 as required). The list of required packages can be found in the `build_msvc\vcpkg-packages.txt` file. The PowerShell command below will work if run from the repository root directory and `vcpkg` is in the path. Alternatively the contents of the packages text file can be pasted in place of the `Get-Content` cmdlet. ``` - PS >.\vcpkg install --triplet x64-windows-static boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb rapidcheck double-conversion +PS >.\vcpkg install --triplet x64-windows-static $(Get-Content -Path build_msvc\vcpkg-packages.txt).split() ``` -- Use Python to generate *.vcxproj from Makefile +- Use Python to generate `*.vcxproj` from Makefile ``` - PS >py -3 msvc-autogen.py +PS >py -3 msvc-autogen.py ``` - An optional step is to adjust the settings in the build_msvc directory and the common.init.vcxproj file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. @@ -74,13 +68,13 @@ The instructions below use `vcpkg` to install the dependencies. - Build with Visual Studio 2017 or msbuild. ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build +msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build ``` - Build with Visual Studio 2019 or msbuild. ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v142 /t:build +msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` AppVeyor diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index fdeec55ee8..dcfc1e1b33 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -56,6 +56,7 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> @@ -69,6 +70,7 @@ </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> <ResourceCompile> <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 2fe489239d..e892765fde 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -98,10 +98,6 @@ */ #define HAVE_DECL_DAEMON 0 -/* Define to 1 if you have the declaration of `EVP_MD_CTX_new', and to 0 if - you don't. */ -//#define HAVE_DECL_EVP_MD_CTX_NEW 1 - /* Define to 1 if you have the declaration of `htobe16', and to 0 if you don't. */ #define HAVE_DECL_HTOBE16 0 diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 77f6a5c621..a04a38ff7c 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -39,7 +39,7 @@ <LinkIncremental>true</LinkIncremental> <WholeProgramOptimization>false</WholeProgramOptimization> <UseDebugLibraries>true</UseDebugLibraries> - <PlatformToolset>v141</PlatformToolset> + <PlatformToolset>v142</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> @@ -48,7 +48,7 @@ <LinkIncremental>false</LinkIncremental> <WholeProgramOptimization>true</WholeProgramOptimization> <UseDebugLibraries>false</UseDebugLibraries> - <PlatformToolset>v141</PlatformToolset> + <PlatformToolset>v142</PlatformToolset> <CharacterSet>Unicode</CharacterSet> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir> <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir> diff --git a/build_msvc/common.qt.init.vcxproj b/build_msvc/common.qt.init.vcxproj index e21288e26b..42150a2310 100644 --- a/build_msvc/common.qt.init.vcxproj +++ b/build_msvc/common.qt.init.vcxproj @@ -2,7 +2,7 @@ <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <PropertyGroup Label="QtGlobals"> - <QtBaseDir>C:\Qt5.9.7_ssl_x64_static_vs2017</QtBaseDir> + <QtBaseDir>C:\Qt5.9.8_x64_static_vs2019</QtBaseDir> <QtPluginsLibraryDir>$(QtBaseDir)\plugins</QtPluginsLibraryDir> <QtLibraryDir>$(QtBaseDir)\lib</QtLibraryDir> <QtIncludeDir>$(QtBaseDir)\include</QtIncludeDir> diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 8e54bc7653..bd9b6cc8db 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -10,7 +10,7 @@ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> <ItemGroup> - <ClCompile Include="..\..\src\test\setup_common.cpp" /> + <ClCompile Include="..\..\src\test\util\setup_common.cpp" /> <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" /> <ClCompile Include="..\..\src\qt\test\apptests.cpp" /> <ClCompile Include="..\..\src\qt\test\compattests.cpp" /> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 47e87b59e4..1d05ff6cba 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -9,13 +9,13 @@ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> <ItemGroup> - <ClCompile Include="..\..\src\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\*_properties.cpp" /> + <ClCompile Include="..\..\src\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> - <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> - <ClCompile Include="..\..\src\test\setup_common.cpp" /> <ClCompile Include="..\..\src\test\main.cpp" /> + <ClCompile Include="..\..\src\test\util\*.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> + <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> diff --git a/build_msvc/vcpkg-packages.txt b/build_msvc/vcpkg-packages.txt new file mode 100644 index 0000000000..082a13f1cf --- /dev/null +++ b/build_msvc/vcpkg-packages.txt @@ -0,0 +1 @@ +berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent openssl rapidcheck zeromq double-conversion
\ No newline at end of file diff --git a/ci/README.md b/ci/README.md index 16c481158f..fb1cd7460b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,7 +12,7 @@ To allow for a wide range of tested environments, but also ensure reproducibilit requires `docker` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io ccache bash git +sudo apt install docker.io bash git ``` To run the default test stage, diff --git a/ci/retry/README.md b/ci/retry/README.md index 983a498070..1b03c652db 100644 --- a/ci/retry/README.md +++ b/ci/retry/README.md @@ -32,8 +32,8 @@ Help: -v, --verbose Verbose output -t, --tries=# Set max retries: Default 10 -s, --sleep=secs Constant sleep amount (seconds) - -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 - -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -m, --min=secs Exponential Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponential Backoff: maximum sleep amount (seconds): Default 60 -f, --fail="script +cmds" Fail Script: run in case of final failure ### Examples diff --git a/ci/retry/retry b/ci/retry/retry index 0e5f6e9701..3c06519dbd 100755 --- a/ci/retry/retry +++ b/ci/retry/retry @@ -17,7 +17,7 @@ __log_out() { echo "$1" 1>&2 } -# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND +# Parameters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND retry() { local max_tries="$1"; shift @@ -83,8 +83,8 @@ Usage: $retry [options] -- execute command -v, --verbose Verbose output -t, --tries=# Set max retries: Default 10 -s, --sleep=secs Constant sleep amount (seconds) - -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 - -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -m, --min=secs Exponential Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponential Backoff: maximum sleep amount (seconds): Default 60 -f, --fail="script +cmds" Fail Script: run in case of final failure EOF } diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8bc50da2c3..c65054cf9e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -21,7 +21,10 @@ echo "Fallback to default values in env (if not yet set)" export MAKEJOBS=${MAKEJOBS:--j4} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} -export HOST=${HOST:-x86_64-unknown-linux-gnu} +# What host to compile for. See also ./depends/README.md +# Tests that need cross-compilation export the appropriate HOST. +# Tests that run natively guess the host +export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} # Whether to prefer BusyBox over GNU utilities export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 9335f0b337..6e2542584c 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -7,11 +7,16 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf -export QEMU_USER_CMD="qemu-arm -L /usr/arm-linux-gnueabihf/" -export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user" +# The host arch is unknown, so we run the tests through qemu. +# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. +export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-arm -L /usr/arm-linux-gnueabihf/"}" +# We don't know whether the host can run the cross compiled binaries. To run them, either qemu-user or libc6:armhf for +# the target is required, so install both. +export DPKG_ADD_ARCH="armhf" +export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=false +export RUN_FUNCTIONAL_TESTS=true export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default diff --git a/ci/test/00_setup_env_i686.sh b/ci/test/00_setup_env_i686.sh index 63068dc95d..6df65dd4a0 100644 --- a/ci/test/00_setup_env_i686.sh +++ b/ci/test/00_setup_env_i686.sh @@ -7,8 +7,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu -export DEP_OPTS="PROTOBUF=1" export PACKAGES="g++-multilib python3-zmq" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_mac_functional.sh b/ci/test/00_setup_env_mac_host.sh index e9e68c47a8..aa68a5700e 100644 --- a/ci/test/00_setup_env_mac_functional.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin14 -export BREW_PACKAGES="automake berkeley-db4 libtool boost miniupnpc pkg-config protobuf qt qrencode python3 ccache zeromq" +export BREW_PACKAGES="automake berkeley-db4 libtool boost miniupnpc pkg-config qt qrencode python3 ccache zeromq" export PIP_PACKAGES="zmq" export RUN_CI_ON_HOST=true export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=true +export RUN_FUNCTIONAL_TESTS=false export GOAL="install" -export BITCOIN_CONFIG="--enable-gui --enable-bip70 --enable-reduce-exports --enable-werror" +export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" # Run without depends export NO_DEPENDS=1 export OSX_SDK="" diff --git a/ci/test/00_setup_env_amd64_asan.sh b/ci/test/00_setup_env_native_asan.sh index 46b870e145..293f37fedf 100644 --- a/ci/test/00_setup_env_amd64_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" diff --git a/ci/test/00_setup_env_amd64_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 7bdfc2c320..ed331e5069 100644 --- a/ci/test/00_setup_env_amd64_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -6,11 +6,11 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu -export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" +export PACKAGES="clang-8 llvm-8 python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang-8 CXX=clang++-8" +# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64 diff --git a/ci/test/00_setup_env_amd64_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index d5a2ba3111..53348559be 100644 --- a/ci/test/00_setup_env_amd64_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="python3-zmq" export DEP_OPTS="NO_WALLET=1" export GOAL="install" diff --git a/ci/test/00_setup_env_amd64_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 55820ea835..622ec3cfe1 100644 --- a/ci/test/00_setup_env_amd64_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash diff --git a/ci/test/00_setup_env_amd64_trusty.sh b/ci/test/00_setup_env_native_trusty.sh index 51e98788c7..fbf1488f07 100644 --- a/ci/test/00_setup_env_amd64_trusty.sh +++ b/ci/test/00_setup_env_native_trusty.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export DOCKER_NAME_TAG=ubuntu:14.04 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_amd64_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 82ac988c41..afc082d6cf 100644 --- a/ci/test/00_setup_env_amd64_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-unknown-linux-gnu export DOCKER_NAME_TAG=ubuntu:16.04 export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 1578e3c0b4..271ae82e5c 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -33,7 +33,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then fi mkdir -p "${BASE_SCRATCH_DIR}" -ccache echo "Creating ccache dir if it didn't already exist" +mkdir -p "${CCACHE_DIR}" if [ ! -d ${DIR_QA_ASSETS} ]; then git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} @@ -41,7 +41,7 @@ fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" -export ASAN_OPTIONS="" +export ASAN_OPTIONS="detect_stack_use_after_return=1" export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" @@ -76,6 +76,9 @@ else DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) fi +if [ -n "$DPKG_ADD_ARCH" ]; then + DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" +fi if [ "$TRAVIS_OS_NAME" != "osx" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 048fe06c84..7ac4907d32 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -32,7 +32,7 @@ fi if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - bash -c "while sleep 500; do echo .; done" & # Print dots in case the unit tests take a long time to run + bash -c "${CI_WAIT}" & # Print dots in case the unit tests take a long time to run DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi diff --git a/configure.ac b/configure.ac index 715b65f091..0f31bbaee5 100644 --- a/configure.ac +++ b/configure.ac @@ -226,13 +226,16 @@ AC_ARG_ENABLE([zmq], [disable ZMQ notifications])], [use_zmq=$enableval], [use_zmq=yes]) + AC_ARG_ENABLE([bip70], [AS_HELP_STRING([--enable-bip70], - [enable BIP70 (payment protocol) support in the GUI (default is to disable)])], + [BIP70 (payment protocol) support in the GUI (no longer supported)])], [enable_bip70=$enableval], [enable_bip70=no]) -AC_ARG_WITH([protoc-bindir],[AS_HELP_STRING([--with-protoc-bindir=BIN_DIR],[specify protoc bin path])], [protoc_bin_path=$withval], []) +if test x$enable_bip70 != xno; then + AC_MSG_ERROR([BIP70 is no longer supported!]) +fi AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], @@ -326,6 +329,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -338,6 +342,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all ## unknown options if any other warning is produced. Test the -Wfoo case, and @@ -594,6 +599,7 @@ case $host in ;; *android*) dnl make sure android stays above linux for hosts like *linux-android* + TARGET_OS=android LEVELDB_TARGET_FLAGS="-DOS_ANDROID" ;; *linux*) @@ -784,7 +790,7 @@ if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) # FD_ZERO may be dependent on a declaration of memcpy, e.g. in SmartOS # check that it fails to build without memcpy, then that it builds with @@ -944,6 +950,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> [ AC_MSG_RESULT(no)] ) +AC_MSG_CHECKING(for sysctl) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> + #include <sys/sysctl.h>]], + [[ static const int name[2] = {CTL_KERN, KERN_VERSION}; + #ifdef __linux__ + #error "Don't use sysctl on Linux, it's deprecated even when it works" + #endif + sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], + [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT(no)] +) + AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #include <sys/sysctl.h>]], @@ -1237,11 +1255,7 @@ if test x$use_pkgconfig = xyes; then m4_ifdef( [PKG_CHECK_MODULES], [ - PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) - if test x$enable_bip70 != xno; then - BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [have_protobuf=no])]) - fi if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi @@ -1267,9 +1281,6 @@ else AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) - AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) - AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) - if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) @@ -1301,22 +1312,12 @@ else esac fi - if test x$enable_bip70 != xno; then - BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], [have_protobuf=no])) - fi if test x$use_qr != xno; then BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) fi fi -save_CXXFLAGS="${CXXFLAGS}" -CXXFLAGS="${CXXFLAGS} ${CRYPTO_CFLAGS} ${SSL_CFLAGS}" -AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT -#include <openssl/x509_vfy.h> -]) -CXXFLAGS="${save_CXXFLAGS}" - dnl RapidCheck property-based testing enable_property_tests=no @@ -1381,12 +1382,6 @@ AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) - -if test x$have_protobuf != xno && - test x$enable_bip70 != xno; then - BITCOIN_QT_PATH_PROGS([PROTOC], [protoc],$protoc_bin_path) -fi - AC_MSG_CHECKING([whether to build bitcoind]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) @@ -1508,23 +1503,6 @@ if test x$bitcoin_enable_qt != xno; then else AC_MSG_RESULT([no]) fi - - AC_MSG_CHECKING([whether to build BIP70 support]) - if test x$have_protobuf = xno; then - if test x$enable_bip70 = xyes; then - AC_MSG_ERROR(protobuf missing) - fi - enable_bip70=no - AC_MSG_RESULT(no) - else - if test x$enable_bip70 != xno; then - AC_DEFINE([ENABLE_BIP70],[1],[Define if BIP70 support should be compiled in]) - enable_bip70=yes - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi - fi fi AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) @@ -1557,7 +1535,6 @@ AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) -AM_CONDITIONAL([ENABLE_BIP70],[test x$enable_bip70 = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) @@ -1623,11 +1600,9 @@ AC_SUBST(LEVELDB_TARGET_FLAGS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(CRYPTO_LIBS) -AC_SUBST(SSL_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) -AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) @@ -1682,7 +1657,6 @@ echo "Options used to compile and link:" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then - echo " with bip70 = $enable_bip70" echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" diff --git a/contrib/bitcoin-cli.bash-completion b/contrib/bitcoin-cli.bash-completion index f4cac84182..f7f12a2773 100644 --- a/contrib/bitcoin-cli.bash-completion +++ b/contrib/bitcoin-cli.bash-completion @@ -17,13 +17,6 @@ _bitcoin_rpc() { $bitcoin_cli "${rpcargs[@]}" "$@" } -# Add wallet accounts to COMPREPLY -_bitcoin_accounts() { - local accounts - accounts=$(_bitcoin_rpc listaccounts | awk -F '"' '{ print $2 }') - COMPREPLY=( "${COMPREPLY[@]}" $( compgen -W "$accounts" -- "$cur" ) ) -} - _bitcoin_cli() { local cur prev words=() cword local bitcoin_cli @@ -60,10 +53,9 @@ _bitcoin_cli() { if ((cword > 3)); then case ${words[cword-3]} in addmultisigaddress) - _bitcoin_accounts return 0 ;; - getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaccount|listreceivedbyaddress|listsinceblock) + getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock) COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) return 0 ;; @@ -80,14 +72,10 @@ _bitcoin_cli() { COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) ) return 0 ;; - fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listaccounts|listreceivedbyaccount|listreceivedbyaddress|sendrawtransaction) + fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction) COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) return 0 ;; - move|setaccount) - _bitcoin_accounts - return 0 - ;; esac fi @@ -96,12 +84,11 @@ _bitcoin_cli() { _filedir return 0 ;; - getaddednodeinfo|getrawmempool|lockunspent|setgenerate) + getaddednodeinfo|getrawmempool|lockunspent) COMPREPLY=( $( compgen -W "true false" -- "$cur" ) ) return 0 ;; - getaccountaddress|getaddressesbyaccount|getbalance|getnewaddress|getreceivedbyaccount|listtransactions|move|sendfrom|sendmany) - _bitcoin_accounts + getbalance|getnewaddress|listtransactions|sendmany) return 0 ;; esac diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion index cccd4bde0d..da869fa2c3 100644 --- a/contrib/bitcoind.bash-completion +++ b/contrib/bitcoind.bash-completion @@ -15,7 +15,7 @@ _bitcoind() { _get_comp_words_by_ref -n = cur prev words cword case "$cur" in - -conf=*|-pid=*|-loadblock=*|-rootcertificates=*|-rpccookiefile=*|-wallet=*) + -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*) cur="${cur#*=}" _filedir return 0 diff --git a/contrib/devtools/utxo_snapshot.sh b/contrib/devtools/utxo_snapshot.sh new file mode 100755 index 0000000000..dee25ff67b --- /dev/null +++ b/contrib/devtools/utxo_snapshot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +export LC_ALL=C + +set -ueo pipefail + +if (( $# < 3 )); then + echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>' + echo + echo " if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the " + echo " expected assumeutxo hash" + echo + echo 'Examples:' + echo + echo " ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata" + echo ' ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli' + exit 1 +fi + +GENERATE_AT_HEIGHT="${1}"; shift; +OUTPUT_PATH="${1}"; shift; +# Most of the calls we make take a while to run, so pad with a lengthy timeout. +BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999" + +# Block we'll invalidate/reconsider to rewind/fast-forward the chain. +PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) ) + +(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while") +${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}" + +if [[ "${OUTPUT_PATH}" = "-" ]]; then + (>&2 echo "Generating txoutset info...") + ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g' +else + (>&2 echo "Generating UTXO snapshot...") + ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}" +fi + +(>&2 echo "Restoring chain to original height; this may take a while") +${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e0b9f74397..c5b4ac1e83 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "curl" - "g++-aarch64-linux-gnu" @@ -59,11 +59,12 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -77,7 +78,7 @@ script: | then echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 4cfca403b1..9957fc0514 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -4,7 +4,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "faketime" remotes: @@ -24,7 +24,7 @@ script: | for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index a563bef778..79b1b22cdf 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "ca-certificates" - "curl" @@ -55,11 +55,12 @@ script: | export ZERO_AR_DATE=1 + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -71,7 +72,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index b151caa3cc..1f5b9d4507 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -4,7 +4,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "libssl-dev" - "autoconf" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index d600af84fb..4259a10e14 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "amd64" +- "linux64" packages: - "curl" - "g++" @@ -48,11 +48,12 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi + # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -64,7 +65,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} @@ -79,7 +80,7 @@ script: | for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} @@ -145,10 +146,7 @@ script: | make ${MAKEOPTS} -C src check-security make deploy make install DESTDIR=${INSTALLPATH} - ( - SETUP_EXE="$(basename "$(echo ./*-setup.exe)")" - cp -f "$SETUP_EXE" "${OUTDIR}/${SETUP_EXE/%-setup.exe/-setup-unsigned.exe}" - ) + cp -f --target-directory="${OUTDIR}" ./bitcoin-*-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete diff --git a/contrib/gitian-keys/keys.txt b/contrib/gitian-keys/keys.txt index 9222a40b17..ba3036a89f 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/gitian-keys/keys.txt @@ -27,6 +27,7 @@ D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy 37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille (Location: Leuven, Belgium) 133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille +A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index ca11d7a0f0..23b656cad7 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -106,7 +106,7 @@ chain for " target " development.")) (base-libc glibc-2.27) (base-gcc (make-gcc-rpath-link (make-ssp-fixed-gcc gcc-9)))) - "Convienience wrapper around MAKE-CROSS-TOOLCHAIN with default values + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target base-gcc-for-libc diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 0f6d993fd2..3c485c1df6 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -6,7 +6,14 @@ # Example use: # $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin # $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ -# --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +# --show-leak-kinds=all src/test/test_bitcoin +# +# To create suppressions for found issues, use the --gen-suppressions=all option: +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all --gen-suppressions=all --show-reachable=yes \ +# --error-limit=no src/test/test_bitcoin +# +# Note that suppressions may depend on OS and/or library versions. { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -27,6 +34,14 @@ fun:__log_put_record } { + Suppress libdb warning + Memcheck:Param + pwrite64(buf) + fun:pwrite + fun:__os_io + obj:*/libdb_cxx-*.so +} +{ Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 Memcheck:Leak match-leak-kinds: reachable @@ -41,3 +56,57 @@ ... fun:_ZN7leveldbL14InitDefaultEnvEv } +{ + Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626) + Memcheck:Addr16 + fun:__wcsnlen_sse4_1 + fun:wcsnrtombs +} +{ + Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9) + Memcheck:Cond + fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE + fun:_ZN5boost10filesystem6detail28directory_iterator_constructERNS0_18directory_iteratorERKNS0_4pathEPNS_6system10error_codeE + obj:*/libboost_filesystem.so.* +} +{ + Suppress boost::filesystem warning (could be related: https://stackoverflow.com/questions/9830182/function-boostfilesystemcomplete-being-reported-as-possible-memory-leak-by-v) + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_ +} +{ + Suppress boost still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_M_construct_aux<char*> + fun:_M_construct<char*> + fun:basic_string + fun:path +} +{ + Suppress LogInstance still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_Z11LogInstancev +} +{ + Suppress secp256k1_context_create still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:secp256k1_context_create +} +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} diff --git a/depends/Makefile b/depends/Makefile index 25ff135ea6..cec99777ff 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -9,7 +9,6 @@ WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs NO_QT ?= -PROTOBUF ?= RAPIDCHECK ?= NO_WALLET ?= NO_ZMQ ?= @@ -59,6 +58,11 @@ full_host_os:=$(subst $(host_arch)-$(host_vendor)-,,$(canonical_host)) host_os:=$(findstring linux,$(full_host_os)) host_os+=$(findstring darwin,$(full_host_os)) host_os+=$(findstring mingw32,$(full_host_os)) + +ifeq (android,$(findstring android,$(full_host_os))) +host_os:=android +endif + host_os:=$(strip $(host_os)) ifeq ($(host_os),) host_os=$(full_host_os) @@ -101,17 +105,11 @@ wallet_packages_$(NO_WALLET) = $(wallet_packages) upnp_packages_$(NO_UPNP) = $(upnp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) -protobuf_packages_$(PROTOBUF) = $(protobuf_packages) rapidcheck_packages_$(RAPIDCHECK) = $(rapidcheck_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) -ifeq ($(protobuf_packages_),) -native_packages += $(protobuf_native_packages) -packages += $(protobuf_packages) -endif - ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif @@ -157,7 +155,6 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ -e 's|@no_qt@|$(NO_QT)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ - -e 's|@enable_bip70@|$(PROTOBUF)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ -e 's|@debug@|$(DEBUG)|' \ diff --git a/depends/README.md b/depends/README.md index cfb9bbfeb0..e5e2a8a653 100644 --- a/depends/README.md +++ b/depends/README.md @@ -30,8 +30,12 @@ Common `host-platform-triplets` for cross compilation are: - `aarch64-linux-gnu` for Linux ARM 64 bit - `riscv32-linux-gnu` for Linux RISC-V 32 bit - `riscv64-linux-gnu` for Linux RISC-V 64 bit +- `armv7a-linux-android` for Android ARM 32 bit +- `aarch64-linux-android` for Android ARM 64 bit +- `i686-linux-android` for Android x86 32 bit +- `x86_64-linux-android` for Android x86 64 bit -No other options are needed, the paths are automatically configured. +The paths are automatically configured and no other options are needed unless targeting [Android](#Android). ### Install the required dependencies: Ubuntu & Debian @@ -77,7 +81,6 @@ The following can be set when running make: make FOO=bar NO_UPNP: Don't download/build/cache packages needed for enabling upnp DEBUG: disable some optimizations and enable more runtime checking RAPIDCHECK: build rapidcheck (experimental, requires cmake) - PROTOBUF: build protobuf (used for deprecated BIP70 support) HOST_ID_SALT: Optional salt to use when generating host package ids BUILD_ID_SALT: Optional salt to use when generating build package ids @@ -91,6 +94,19 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-win: run 'make download-win' to fetch all sources needed for win builds download-linux: run 'make download-linux' to fetch all sources needed for linux builds + +### Android + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/depends/config.site.in b/depends/config.site.in index d0d36641c4..dee568bc25 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -16,10 +16,6 @@ fi if test -z $with_qt_bindir && test -z "@no_qt@"; then with_qt_bindir=$depends_prefix/native/bin fi -if test -z $with_protoc_bindir && test -z "@no_qt@"; then - with_protoc_bindir=$depends_prefix/native/bin -fi - if test -z $enable_wallet && test -n "@no_wallet@"; then enable_wallet=no @@ -37,10 +33,6 @@ if test -z $enable_zmq && test -n "@no_zmq@"; then enable_zmq=no fi -if test -n $enable_bip70 && test -n "@enable_bip70@"; then - enable_bip70=yes -fi - if test x@host_os@ = xdarwin; then BREW=no PORT=no diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk new file mode 100644 index 0000000000..969ec2a1cb --- /dev/null +++ b/depends/hosts/android.mk @@ -0,0 +1,11 @@ +ifeq ($(HOST),armv7a-linux-android) +android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar +android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ +android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib +else +android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar +android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ +android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib +endif diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 5df49b2af8..d360bb5ba6 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -10,18 +10,22 @@ $(package)_config_opts_debug=variant=debug $(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam $(package)_config_opts+=threading=multi link=static -sNO_BZIP2=1 -sNO_ZLIB=1 $(package)_config_opts_linux=threadapi=pthread runtime-link=shared -$(package)_config_opts_darwin=--toolset=darwin-4.2.1 runtime-link=shared +$(package)_config_opts_darwin=--toolset=clang-darwin runtime-link=shared $(package)_config_opts_mingw32=binary-format=pe target-os=windows threadapi=win32 runtime-link=static $(package)_config_opts_x86_64_mingw32=address-model=64 $(package)_config_opts_i686_mingw32=address-model=32 $(package)_config_opts_i686_linux=address-model=32 architecture=x86 +$(package)_config_opts_i686_android=address-model=32 +$(package)_config_opts_aarch64_android=address-model=64 +$(package)_config_opts_x86_64_android=address-model=64 +$(package)_config_opts_armv7a_android=address-model=32 $(package)_toolset_$(host_os)=gcc $(package)_archiver_$(host_os)=$($(package)_ar) -$(package)_toolset_darwin=darwin -$(package)_archiver_darwin=$($(package)_libtool) +$(package)_toolset_darwin=clang-darwin $(package)_config_libraries=chrono,filesystem,system,thread,test $(package)_cxxflags=-std=c++11 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC +$(package)_cxxflags_android=-fPIC endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 300c806a8d..b4333abaf0 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,16 +3,24 @@ $(package)_version=2.1.8-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ $(package)_file_name=release-$($(package)_version).tar.gz $(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d +$(package)_patches=fix_android_arc4random_addrandom.patch -define $(package)_preprocess_cmds - ./autogen.sh -endef +ifneq (,$(findstring android,$(host))) + define $(package)_preprocess_cmds + ./autogen.sh && patch -p1 < $($(package)_patch_dir)/fix_android_arc4random_addrandom.patch + endef +else + define $(package)_preprocess_cmds + ./autogen.sh + endef +endif define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic + $(package)_config_opts_android=--with-pic endef define $(package)_config_cmds diff --git a/depends/packages/native_protobuf.mk b/depends/packages/native_protobuf.mk deleted file mode 100644 index 1de8c37d36..0000000000 --- a/depends/packages/native_protobuf.mk +++ /dev/null @@ -1,25 +0,0 @@ -package=native_protobuf -$(package)_version=2.6.1 -$(package)_download_path=https://github.com/google/protobuf/releases/download/v$($(package)_version) -$(package)_file_name=protobuf-$($(package)_version).tar.bz2 -$(package)_sha256_hash=ee445612d544d885ae240ffbcbf9267faa9f593b7b101f21d58beceb92661910 - -define $(package)_set_vars -$(package)_config_opts=--disable-shared --without-zlib -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -C src protoc -endef - -define $(package)_stage_cmds - $(MAKE) -C src DESTDIR=$($(package)_staging_dir) install-strip -endef - -define $(package)_postprocess_cmds - rm -rf lib include -endef diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk index 3e8a22a1b0..e3b3647dd9 100644 --- a/depends/packages/openssl.mk +++ b/depends/packages/openssl.mk @@ -58,6 +58,11 @@ $(package)_config_opts_riscv64_linux=linux-generic64 $(package)_config_opts_x86_64_darwin=darwin64-x86_64-cc $(package)_config_opts_x86_64_mingw32=mingw64 $(package)_config_opts_i686_mingw32=mingw +$(package)_config_opts_android=-fPIC +$(package)_config_opts_aarch64_android=linux-generic64 +$(package)_config_opts_x86_64_android=linux-generic64 +$(package)_config_opts_armv7a_android=linux-generic32 +$(package)_config_opts_i686_android=linux-generic32 endef define $(package)_preprocess_cmds @@ -71,7 +76,7 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) -j1 build_libs libcrypto.pc libssl.pc openssl.pc + $(MAKE) -j1 build_crypto libcrypto.pc libssl.pc openssl.pc endef define $(package)_stage_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 667fde5271..09734c7e6f 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,11 +1,9 @@ packages:=boost openssl libevent -protobuf_native_packages = native_protobuf -protobuf_packages = protobuf - qt_packages = qrencode zlib qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig +qt_android_packages=qt rapidcheck_packages = rapidcheck diff --git a/depends/packages/protobuf.mk b/depends/packages/protobuf.mk deleted file mode 100644 index 3661a16631..0000000000 --- a/depends/packages/protobuf.mk +++ /dev/null @@ -1,34 +0,0 @@ -package=protobuf -$(package)_version=$(native_$(package)_version) -$(package)_download_path=$(native_$(package)_download_path) -$(package)_file_name=$(native_$(package)_file_name) -$(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) -$(package)_cxxflags=-std=c++11 - -define $(package)_set_vars - $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc --disable-dependency-tracking - $(package)_config_opts_linux=--with-pic -endef - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . &&\ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub gtest/build-aux -endef - -define $(package)_config_cmds - $($(package)_autoconf) -endef - -define $(package)_build_cmds - $(MAKE) -C src libprotobuf.la -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install-libLTLIBRARIES install-nobase_includeHEADERS &&\ - $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA -endef - -define $(package)_postprocess_cmds - rm lib/libprotoc.a lib/*.la -endef diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 21570726ff..d1687883bc 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -9,6 +9,7 @@ $(package)_config_opts=--disable-shared --without-tools --without-tests --disabl $(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_linux=--with-pic +$(package)_config_opts_android=--with-pic endef define $(package)_preprocess_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index f4832b6168..6ac6ba6838 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -4,11 +4,11 @@ $(package)_download_path=https://download.qt.io/official_releases/qt/5.9/$($(pac $(package)_suffix=opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=9b9dec1f67df1f94bce2955c5604de992d529dde72050239154c56352da0907d -$(package)_dependencies=openssl zlib +$(package)_dependencies=zlib $(package)_linux_dependencies=freetype fontconfig libxcb $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch +$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=fb5a47799754af73d3bf501fe513342cfe2fc37f64e80df5533f6110e804220c @@ -42,9 +42,11 @@ $(package)_config_opts += -no-linuxfb $(package)_config_opts += -no-libjpeg $(package)_config_opts += -no-libudev $(package)_config_opts += -no-mtdev +$(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations $(package)_config_opts += -no-qml-debug +$(package)_config_opts += -no-securetransport $(package)_config_opts += -no-sql-db2 $(package)_config_opts += -no-sql-ibase $(package)_config_opts += -no-sql-oci @@ -59,7 +61,6 @@ $(package)_config_opts += -no-xinput2 $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -opensource -$(package)_config_opts += -openssl-linked $(package)_config_opts += -optimized-qmake $(package)_config_opts += -pch $(package)_config_opts += -pkg-config @@ -127,6 +128,26 @@ $(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 $(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ $(package)_config_opts_riscv64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_mingw32 = -no-opengl -xplatform win32-g++ -device-option CROSS_COMPILE="$(host)-" + +$(package)_config_opts_android = -xplatform android-clang +$(package)_config_opts_android += -android-sdk $(ANDROID_SDK) +$(package)_config_opts_android += -android-ndk $(ANDROID_NDK) +$(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LEVEL) +$(package)_config_opts_android += -device-option CROSS_COMPILE="$(host)-" +$(package)_config_opts_android += -egl +$(package)_config_opts_android += -qpa xcb +$(package)_config_opts_android += -no-eglfs +$(package)_config_opts_android += -opengl es2 +$(package)_config_opts_android += -qt-freetype +$(package)_config_opts_android += -no-fontconfig +$(package)_config_opts_android += -L $(host_prefix)/lib +$(package)_config_opts_android += -I $(host_prefix)/include + +$(package)_config_opts_aarch64_android += -android-arch arm64-v8a +$(package)_config_opts_armv7a_android += -android-arch armeabi-v7a +$(package)_config_opts_x86_64_android += -android-arch x86_64 +$(package)_config_opts_i686_android += -android-arch i686 + $(package)_build_env = QT_RCC_TEST=1 $(package)_build_env += QT_RCC_SOURCE_DATE_OVERRIDE=1 endef @@ -169,6 +190,8 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch &&\ patch -p1 -i $($(package)_patch_dir)/fix_rcc_determinism.patch &&\ patch -p1 -i $($(package)_patch_dir)/xkb-default.patch &&\ + patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch &&\ + patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch &&\ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ @@ -179,6 +202,8 @@ define $(package)_preprocess_cmds sed -i.old "s|QMAKE_CFLAGS += |!host_build: QMAKE_CFLAGS = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "s|QMAKE_CXXFLAGS += |!host_build: QMAKE_CXXFLAGS = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \ sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \ + sed -i.old "s|QMAKE_CC = clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ + sed -i.old "s|QMAKE_CXX = clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index ce85c437bb..6f35ede248 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -11,6 +11,7 @@ define $(package)_set_vars $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic + $(package)_config_opts_android=--with-pic $(package)_cxxflags=-std=c++11 endef diff --git a/depends/packages/zlib.mk b/depends/packages/zlib.mk index 168f85e65e..acb02020a8 100644 --- a/depends/packages/zlib.mk +++ b/depends/packages/zlib.mk @@ -11,6 +11,7 @@ $(package)_config_opts+=RANLIB="$($(package)_ranlib)" $(package)_config_opts+=AR="$($(package)_ar)" $(package)_config_opts_darwin+=AR="$($(package)_libtool)" $(package)_config_opts_darwin+=ARFLAGS="-o" +$(package)_config_opts_android+=CHOST=$(host) endef # zlib has its own custom configure script that takes in options like CC, diff --git a/depends/patches/libevent/fix_android_arc4random_addrandom.patch b/depends/patches/libevent/fix_android_arc4random_addrandom.patch new file mode 100644 index 0000000000..5bcc64bef6 --- /dev/null +++ b/depends/patches/libevent/fix_android_arc4random_addrandom.patch @@ -0,0 +1,68 @@ +From cadae3ab7abf45e61ecae8aac39d97d1f3cbd336 Mon Sep 17 00:00:00 2001 +From: Lawrence Nahum <lawrence@greenaddress.it> +Date: Sun, 3 Dec 2017 22:56:09 +0100 +Subject: [PATCH] fixup + +--- + configure.ac | 1 + + evutil_rand.c | 3 +++ + include/event2/util.h | 4 ++-- + 3 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/configure.ac b/configure.ac +index 7528d37..3bb2121 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -341,6 +341,7 @@ dnl Checks for library functions. + AC_CHECK_FUNCS([ \ + accept4 \ + arc4random \ ++ arc4random_addrandom \ + arc4random_buf \ + eventfd \ + epoll_create1 \ +diff --git a/evutil_rand.c b/evutil_rand.c +index 046a14b..3f0bf2c 100644 +--- a/evutil_rand.c ++++ b/evutil_rand.c +@@ -191,6 +191,7 @@ evutil_secure_rng_get_bytes(void *buf, size_t n) + { + ev_arc4random_buf(buf, n); + } ++#ifdef HAVE_ARC4RANDOM_ADDRANDOM + + void + evutil_secure_rng_add_bytes(const char *buf, size_t n) +@@ -199,6 +200,8 @@ evutil_secure_rng_add_bytes(const char *buf, size_t n) + n>(size_t)INT_MAX ? INT_MAX : (int)n); + } + ++#endif ++ + void + evutil_free_secure_rng_globals_(void) + { +diff --git a/include/event2/util.h b/include/event2/util.h +index dd4bbb6..a9a169d 100644 +--- a/include/event2/util.h ++++ b/include/event2/util.h +@@ -841,7 +841,7 @@ int evutil_secure_rng_init(void); + */ + EVENT2_EXPORT_SYMBOL + int evutil_secure_rng_set_urandom_device_file(char *fname); +- ++#ifdef HAVE_ARC4RANDOM_ADDRANDOM + /** Seed the random number generator with extra random bytes. + + You should almost never need to call this function; it should be +@@ -858,7 +858,7 @@ int evutil_secure_rng_set_urandom_device_file(char *fname); + */ + EVENT2_EXPORT_SYMBOL + void evutil_secure_rng_add_bytes(const char *dat, size_t datlen); +- ++#endif + #ifdef __cplusplus + } + #endif +-- +2.14.3 diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch new file mode 100644 index 0000000000..2f6ff00f40 --- /dev/null +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -0,0 +1,18 @@ +--- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp ++++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp +@@ -890,6 +890,14 @@ + __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); + return -1; + } ++ ++ const jint ret = QT_PREPEND_NAMESPACE(QtAndroidPrivate::initJNI(vm, env)); ++ if (ret != 0) ++ { ++ __android_log_print(ANDROID_LOG_FATAL, "Qt", "initJNI failed"); ++ return ret; ++ } ++ + QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); + + m_javaVM = vm; + diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch new file mode 100644 index 0000000000..13bfff9776 --- /dev/null +++ b/depends/patches/qt/fix_android_qmake_conf.patch @@ -0,0 +1,20 @@ +--- old/qtbase/mkspecs/android-clang/qmake.conf ++++ new/qtbase/mkspecs/android-clang/qmake.conf +@@ -30,7 +30,7 @@ + QMAKE_CFLAGS += -target mips64el-none-linux-android + + QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH +-QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a ++QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -nostdlib++ + QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ + -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ + -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ +@@ -40,7 +40,7 @@ + ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH + + ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so +-ANDROID_CXX_STL_LIBS = -lc++ ++ANDROID_CXX_STL_LIBS = -lc++_shared + + QMAKE_ARM_CFLAGS_RELEASE = -Oz + QMAKE_ARM_CFLAGS_RELEASE_WITH_DEBUGINFO = -g -Oz diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 3b8b0db162..a63391e01a 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -50,7 +50,7 @@ Given a height: returns hash of block in best-block-chain at height provided. Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name as defined in BIP70 (main, test, regtest) +* chain : (string) current network name (main, test, regtest) * blocks : (numeric) the current number of blocks processed in the server * headers : (numeric) the current number of headers we have validated * bestblockhash : (string) the hash of the currently best block diff --git a/doc/bips.md b/doc/bips.md index 45562cec62..b96862297f 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -19,7 +19,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.19.0**): * [`BIP 65`](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki): The CHECKLOCKTIMEVERIFY softfork was merged in **v0.12.0** ([PR #6351](https://github.com/bitcoin/bitcoin/pull/6351)), and backported to **v0.11.2** and **v0.10.4**. Mempool-only CLTV was added in [PR #6124](https://github.com/bitcoin/bitcoin/pull/6124). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 68`](https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki): Sequence locks have been implemented as of **v0.12.1** ([PR #7184](https://github.com/bitcoin/bitcoin/pull/7184)), and have been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). -* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), and is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). +* [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): + Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). + Support can be optionally disabled at build time since **v0.18.0** ([PR 14451](https://github.com/bitcoin/bitcoin/pull/14451)), + and it is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). + It has been removed as of **v0.20.0** ([PR 17165](https://github.com/bitcoin/bitcoin/pull/17165)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). diff --git a/doc/build-unix.md b/doc/build-unix.md index 069c983e6e..d048bdeff5 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -44,7 +44,6 @@ Optional dependencies: miniupnpc | UPnP Support | Firewall-jumping support libdb4.8 | Berkeley DB | Wallet storage (only needed when wallet enabled) qt | GUI | GUI toolkit (only needed when GUI enabled) - protobuf | Payments in GUI | Data interchange format used for payment protocol (only needed when BIP70 enabled) libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled) univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) @@ -118,10 +117,6 @@ libqrencode (optional) can be installed with: sudo apt-get install libqrencode-dev -protobuf (optional) can be installed with: - - sudo apt-get install libprotobuf-dev protobuf-compiler - Once these are installed, they will be found by configure and a bitcoin-qt executable will be built by default. @@ -150,10 +145,6 @@ libqrencode (optional) can be installed with: sudo dnf install qrencode-devel -protobuf (optional) can be installed with: - - sudo dnf install protobuf-devel - Notes ----- The release is built with GCC and then "strip bitcoind" to strip the debug @@ -163,8 +154,8 @@ symbols, which reduces the executable size by about 90%. miniupnpc --------- -[miniupnpc](http://miniupnp.free.fr/) may be used for UPnP port mapping. It can be downloaded from [here]( -http://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and +[miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( +https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and turned off by default. See the configure options for upnp behavior desired: --without-miniupnpc No UPnP support miniupnp not required diff --git a/doc/build-windows.md b/doc/build-windows.md index 5ca9f98475..bbff638b90 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -8,18 +8,18 @@ The options known to work for building Bitcoin Core on Windows are: * On Linux, using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Bionic 18.04 is required and is the platform used to build the Bitcoin Core Windows release binaries. * On Windows, using [Windows -Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about) and the Mingw-w64 cross compiler tool chain. +Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. +* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). Other options which may work, but which have not been extensively tested are (please contribute instructions): -* On Windows, using a POSIX compatibility layer application such as [cygwin](http://www.cygwin.com/) or [msys2](http://www.msys2.org/). -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). +* On Windows, using a POSIX compatibility layer application such as [cygwin](https://www.cygwin.com/) or [msys2](https://www.msys2.org/). Installing Windows Subsystem for Linux --------------------------------------- With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about). This +Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu-based environment. Within this environment you can cross compile for Windows without the need for a separate Linux VM or server. Note that while WSL can be installed with @@ -28,7 +28,7 @@ tested with Ubuntu. This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. In addition, it is available [only for 64-bit versions of -Windows](https://msdn.microsoft.com/en-us/commandline/wsl/install_guide). +Windows](https://docs.microsoft.com/windows/wsl/install-win10). Full instructions to install WSL are available on the above link. To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: @@ -62,8 +62,7 @@ First, install the general dependencies: sudo apt install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git A host toolchain (`build-essential`) is necessary because some dependency -packages (such as `protobuf`) need to build host utilities that are used in the -build process. +packages need to build host utilities that are used in the build process. See [dependencies.md](dependencies.md) for a complete overview. diff --git a/doc/dependencies.md b/doc/dependencies.md index e5b4084d99..dc88626761 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -19,7 +19,6 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | | Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | @@ -35,7 +34,6 @@ Some dependencies are not needed in all configurations. The following are some f #### Options passed to `./configure` * MiniUPnPc is not needed with `--with-miniupnpc=no`. * Berkeley DB is not needed with `--disable-wallet`. -* protobuf is only needed with `--enable-bip70`. * Qt is not needed with `--without-gui`. * If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. * ZeroMQ is needed only with the `--with-zmq` option. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 9cf4b4b075..e7fd8102a4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -384,7 +384,7 @@ Threads - ThreadScriptCheck : Verifies block scripts. -- ThreadImport : Loads blocks from blk*.dat files or bootstrap.dat. +- ThreadImport : Loads blocks from `blk*.dat` files or `-loadblock=<file>`. - StartNode : Starts other threads. @@ -640,6 +640,28 @@ Strings and formatting - *Rationale*: Bitcoin Core uses tinyformat, which is type safe. Leave them out to avoid confusion. +- Use `.c_str()` sparingly. Its only valid use is to pass C++ strings to C functions that take NULL-terminated + strings. + + - Do not use it when passing a sized array (so along with `.size()`). Use `.data()` instead to get a pointer + to the raw data. + + - *Rationale*: Although this is guaranteed to be safe starting with C++11, `.data()` communicates the intent better. + + - Do not use it when passing strings to `tfm::format`, `strprintf`, `LogPrint[f]`. + + - *Rationale*: This is redundant. Tinyformat handles strings. + + - Do not use it to convert to `QString`. Use `QString::fromStdString()`. + + - *Rationale*: Qt has built-in functionality for converting their string + type from/to C++. No need to roll your own. + + - In cases where do you call `.c_str()`, you might want to additionally check that the string does not contain embedded '\0' characters, because + it will (necessarily) truncate the string. This might be used to hide parts of the string from logging or to circumvent + checks. If a use of strings is sensitive to this, take care to check the string for embedded NULL characters first + and reject it if there are any (see `ParsePrechecks` in `strencodings.cpp` for an example). + Shadowing -------------- diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 3dc6be8b86..50e9251b8d 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -77,13 +77,13 @@ will print an error and suggestion if so. ## libFuzzer -A recent version of `clang`, the address sanitizer and libFuzzer is needed (all +A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). To build all fuzz targets with libFuzzer, run ``` -./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ +./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ make ``` diff --git a/doc/release-notes-15437.md b/doc/release-notes-15437.md index 031e90ccd2..6614207757 100644 --- a/doc/release-notes-15437.md +++ b/doc/release-notes-15437.md @@ -32,3 +32,22 @@ Please use the recommended alternatives if you rely on this deprecated feature: could wait until the transaction has confirmed (taking into account the fee target they set (compare the RPC `estimatesmartfee`)) or listen for the transaction announcement by other network peers to check for propagation. + +The removal of BIP61 REJECT message support also has the following minor RPC +and logging implications: + +* `testmempoolaccept` and `sendrawtransaction` no longer return the P2P REJECT + code when a transaction is not accepted to the mempool. They still return the + verbal reject reason. + +* Log messages that previously reported the REJECT code when a transaction was + not accepted to the mempool now no longer report the REJECT code. The reason + for rejection is still reported. + +Updated RPCs +------------ + +- `testmempoolaccept` and `sendrawtransaction` no longer return the P2P REJECT + code when a transaction is not accepted to the mempool. See the Section + _Removal of reject network messages from Bitcoin Core (BIP61)_ for details on + the removal of BIP61 REJECT message support. diff --git a/doc/release-notes-15954.md b/doc/release-notes-15954.md new file mode 100644 index 0000000000..f4d2c5688c --- /dev/null +++ b/doc/release-notes-15954.md @@ -0,0 +1,4 @@ +Configuration option changes +----------------------------- + +Importing blocks upon startup via the `bootstrap.dat` file no longer occurs by default. The file must now be specified with `-loadblock=<file>`. diff --git a/doc/release-notes-17410.md b/doc/release-notes-17410.md new file mode 100644 index 0000000000..08ed353889 --- /dev/null +++ b/doc/release-notes-17410.md @@ -0,0 +1,5 @@ +Command-line options +-------------------- + +- The `-debug=db` logging category has been renamed to `-debug=walletdb`, to distinguish it from `coindb`. + `-debug=db` has been deprecated and will be removed in the next major release. diff --git a/doc/release-notes-17437.md b/doc/release-notes-17437.md new file mode 100644 index 0000000000..3edfd00a38 --- /dev/null +++ b/doc/release-notes-17437.md @@ -0,0 +1,5 @@ +Low-level RPC Changes +=== + +- The RPC gettransaction, listtransactions and listsinceblock responses now also +includes the height of the block that contains the wallet transaction, if any. diff --git a/doc/release-notes.md b/doc/release-notes.md index ea82962e75..a47c8802b0 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -85,6 +85,7 @@ Wallet ------ - The wallet now by default uses bech32 addresses when using RPC, and creates native segwit change outputs. +- The way that output trust was computed has been fixed in #16766, which impacts confirmed/unconfirmed balance status and coin selection. Low-level changes ================= diff --git a/doc/release-notes/release-notes-0.19.0.md b/doc/release-notes/release-notes-0.19.0.md new file mode 100644 index 0000000000..153da68a6e --- /dev/null +++ b/doc/release-notes/release-notes-0.19.0.md @@ -0,0 +1,1086 @@ +Bitcoin Core version 0.19.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.19.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems using +the Linux kernel, macOS 10.10+, and Windows 7 and newer. It is not recommended +to use Bitcoin Core on unsupported systems. + +Bitcoin Core should also work on most other Unix-like systems but is not +as frequently tested on them. + +From 0.17.0 onwards, macOS <10.10 is no longer supported. 0.17.0 is +built using Qt 5.9.x, which doesn't support versions of macOS older than +10.10. Additionally, Bitcoin Core does not yet change appearance when +macOS "dark mode" is activated. + +Users running macOS Catalina may need to "right-click" and then choose "Open" +to open the Bitcoin Core .dmg. This is due to new signing requirements +imposed by Apple, which the Bitcoin Core project does not yet adhere too. + +Notable changes +=============== + +New user documentation +---------------------- + +- [Reduce memory](https://github.com/bitcoin/bitcoin/blob/master/doc/reduce-memory.md) + suggests configuration tweaks for running Bitcoin Core on systems with + limited memory. (#16339) + +New RPCs +-------- + +- `getbalances` returns an object with all balances (`mine`, + `untrusted_pending` and `immature`). Please refer to the RPC help of + `getbalances` for details. The new RPC is intended to replace + `getbalance`, `getunconfirmedbalance`, and the balance fields in + `getwalletinfo`. These old calls and fields may be removed in a + future version. (#15930, #16239) + +- `setwalletflag` sets and unsets wallet flags that enable or disable + features specific to that existing wallet, such as the new + `avoid_reuse` feature documented elsewhere in these release notes. + (#13756) + +- `getblockfilter` gets the BIP158 filter for the specified block. This + RPC is only enabled if block filters have been created using the + `-blockfilterindex` configuration option. (#14121) + +New settings +------------ + +- `-blockfilterindex` enables the creation of BIP158 block filters for + the entire blockchain. Filters will be created in the background and + currently use about 4 GiB of space. Note: this version of Bitcoin + Core does not serve block filters over the P2P network, although the + local user may obtain block filters using the `getblockfilter` RPC. + (#14121) + +Updated settings +---------------- + +- `whitebind` and `whitelist` now accept a list of permissions to + provide peers connecting using the indicated interfaces or IP + addresses. If no permissions are specified with an address or CIDR + network, the implicit default permissions are the same as previous + releases. See the `bitcoind -help` output for these two options for + details about the available permissions. (#16248) + +- Users setting custom `dbcache` values can increase their setting slightly + without using any more real memory. Recent changes reduced the memory use + by about 9% and made chainstate accounting more accurate (it was underestimating + the use of memory before). For example, if you set a value of "450" before, you + may now set a value of "500" to use about the same real amount of memory. (#16957) + + +Updated RPCs +------------ + +Note: some low-level RPC changes mainly useful for testing are described in the +Low-level Changes section below. + +- `sendmany` no longer has a `minconf` argument. This argument was not + well-specified and would lead to RPC errors even when the wallet's + coin selection succeeded. Users who want to influence coin selection + can use the existing `-spendzeroconfchange`, `-limitancestorcount`, + `-limitdescendantcount` and `-walletrejectlongchains` configuration + arguments. (#15596) + +- `getbalance` and `sendtoaddress`, plus the new RPCs `getbalances` and + `createwallet`, now accept an "avoid_reuse" parameter that controls + whether already used addresses should be included in the operation. + Additionally, `sendtoaddress` will avoid partial spends when + `avoid_reuse` is enabled even if this feature is not already enabled + via the `-avoidpartialspends` command line flag because not doing so + would risk using up the "wrong" UTXO for an address reuse case. + (#13756) + +- RPCs which have an `include_watchonly` argument or `includeWatching` option now default to `true` for watch-only + wallets. Affected RPCs are: `getbalance`, `listreceivedbyaddress`, `listreceivedbylabel`, `listtransactions`, + `listsinceblock`, `gettransaction`, `walletcreatefundedpsbt`, and `fundrawtransaction`. (#16383) + +- `listunspent` now returns a "reused" bool for each output if the + wallet flag "avoid_reuse" is enabled. (#13756) + +- `getblockstats` now uses BlockUndo data instead of the transaction + index, making it much faster, no longer dependent on the `-txindex` + configuration option, and functional for all non-pruned blocks. + (#14802) + +- `utxoupdatepsbt` now accepts a `descriptors` parameter that will fill + out input and output scripts and keys when known. P2SH-witness inputs + will be filled in from the UTXO set when a descriptor is provided that + shows they're spending segwit outputs. See the RPC help text for full + details. (#15427) + +- `sendrawtransaction` and `testmempoolaccept` no longer accept a + `allowhighfees` parameter to fail mempool acceptance if the + transaction fee exceeds the value of the configuration option + `-maxtxfee`. Now there is a hardcoded default maximum feerate that + can be changed when calling either RPC using a `maxfeerate` parameter. + (#15620) + +- `getmempoolancestors`, `getmempooldescendants`, `getmempoolentry`, and + `getrawmempool` no longer return a `size` field unless the + configuration option `-deprecatedrpc=size` is used. Instead a new + `vsize` field is returned with the transaction's virtual size + (consistent with other RPCs such as `getrawtransaction`). (#15637) + +- `getwalletinfo` now includes a `scanning` field that is either `false` + (no scanning) or an object with information about the duration and + progress of the wallet's scanning historical blocks for transactions + affecting its balances. (#15730) + +- `gettransaction` now accepts a third (boolean) argument `verbose`. If + set to `true`, a new `decoded` field will be added to the response containing + the decoded transaction. This field is equivalent to RPC `decoderawtransaction`, + or RPC `getrawtransaction` when `verbose` is passed. (#16185, #16866, #16873) + +- `createwallet` accepts a new `passphrase` parameter. If set, this + will create the new wallet encrypted with the given passphrase. If + unset (the default) or set to an empty string, no encryption will be + used. (#16394) + +- `getchaintxstats` RPC now returns the additional key of + `window_final_block_height`. (#16695) + +- `getmempoolentry` now provides a `weight` field containing the + transaction weight as defined in BIP141. (#16647) + +- The `getnetworkinfo` and `getpeerinfo` commands now contain a new field with decoded network service flags. (#16786) + +- `getdescriptorinfo` now returns an additional `checksum` field + containing the checksum for the unmodified descriptor provided by the + user (that is, before the descriptor is normalized for the + `descriptor` field). (#15986) + +- `joinpsbts` now shuffles the order of the inputs and outputs of the resulting + joined PSBT. Previously, inputs and outputs were added in the order PSBTs were + provided. This made it easy to correlate inputs to outputs, representing a + privacy leak. (#16512) + +- `walletcreatefundedpsbt` now signals BIP125 Replace-by-Fee if the + `-walletrbf` configuration option is set to true. (#15911) + +GUI changes +----------- + +- The GUI wallet now provides bech32 addresses by default. The user may change the address type + during invoice generation using a GUI toggle, or the default address + type may be changed with the `-addresstype` configuration option. + (#15711, #16497) + +- In 0.18.0, a `./configure` flag was introduced to allow disabling BIP70 support in the GUI (support was enabled by default). In 0.19.0, this flag is now __disabled__ by default. If you want to compile Bitcoin Core with BIP70 support in the GUI, you can pass `--enable-bip70` to `./configure`. (#15584) + +Deprecated or removed configuration options +------------------------------------------- + +- `-mempoolreplacement` is removed, although default node behavior + remains the same. This option previously allowed the user to prevent + the node from accepting or relaying BIP125 transaction replacements. + This is different from the remaining configuration option + `-walletrbf`. (#16171) + +Deprecated or removed RPCs +-------------------------- + +- `bumpfee` no longer accepts a `totalFee` option unless the + configuration parameter `deprecatedrpc=totalFee` is specified. This + parameter will be fully removed in a subsequent release. (#15996) + +- `bumpfee` has a new `fee_rate` option as a replacement for the deprecated `totalFee`. (#16727) + +- `generate` is now removed after being deprecated in Bitcoin Core 0.18. + Use the `generatetoaddress` RPC instead. (#15492) + +P2P changes +----------- + +- BIP 61 reject messages were deprecated in v0.18. They are now disabled + by default, but can be enabled by setting the `-enablebip61` command + line option. BIP 61 reject messages will be removed entirely in a + future version of Bitcoin Core. (#14054) + +- To eliminate well-known denial-of-service vectors in Bitcoin Core, + especially for nodes with spinning disks, the default value for the + `-peerbloomfilters` configuration option has been changed to false. + This prevents Bitcoin Core from sending the BIP111 NODE_BLOOM service + flag, accepting BIP37 bloom filters, or serving merkle blocks or + transactions matching a bloom filter. Users who still want to provide + bloom filter support may either set the configuration option to true + to re-enable both BIP111 and BIP37 support or enable just BIP37 + support for specific peers using the updated `-whitelist` and + `-whitebind` configuration options described elsewhere in these + release notes. For the near future, lightweight clients using public + BIP111/BIP37 nodes should still be able to connect to older versions + of Bitcoin Core and nodes that have manually enabled BIP37 support, + but developers of such software should consider migrating to either + using specific BIP37 nodes or an alternative transaction filtering + system. (#16152) + +- By default, Bitcoin Core will now make two additional outbound connections that are exclusively used for block-relay. No transactions or addr messages will be processed on these connections. These connections are designed to add little additional memory or bandwidth resource requirements but should make some partitioning attacks more difficult to carry out. (#15759) + +Miscellaneous CLI Changes +------------------------- + +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to + `chain` and now returns the current network name as defined in BIP70 + (main, test, regtest). (#15566) + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` no longer returns a `bip9_softforks` object. + Instead, information has been moved into the `softforks` object and + an additional `type` field describes how Bitcoin Core determines + whether that soft fork is active (e.g. BIP9 or BIP90). See the RPC + help for details. (#16060) + +- `getblocktemplate` no longer returns a `rules` array containing `CSV` + and `segwit` (the BIP9 deployments that are currently in active + state). (#16060) + +- `getrpcinfo` now returns a `logpath` field with the path to + `debug.log`. (#15483) + +Tests +----- + +- The regression test chain enabled by the `-regtest` command line flag + now requires transactions to not violate standard policy by default. + This is the same default used for mainnet and makes it easier to test + mainnet behavior on regtest. Note that the testnet still allows + non-standard txs by default and that the policy can be locally + adjusted with the `-acceptnonstdtxn` command line flag for both test + chains. (#15891) + +Configuration +------------ + +- A setting specified in the default section but not also specified in a + network-specific section (e.g. testnet) will now produce an error + preventing startup instead of just a warning unless the network is + mainnet. This prevents settings intended for mainnet from being + applied to testnet or regtest. (#15629) + +- On platforms supporting `thread_local`, log lines can be prefixed with + the name of the thread that caused the log. To enable this behavior, + use `-logthreadnames=1`. (#15849) + +Network +------- + +- When fetching a transaction announced by multiple peers, previous versions of + Bitcoin Core would sequentially attempt to download the transaction from each + announcing peer until the transaction is received, in the order that those + peers' announcements were received. In this release, the download logic has + changed to randomize the fetch order across peers and to prefer sending + download requests to outbound peers over inbound peers. This fixes an issue + where inbound peers could prevent a node from getting a transaction. + (#14897, #15834) + +- If a Tor hidden service is being used, Bitcoin Core will be bound to + the standard port 8333 even if a different port is configured for + clearnet connections. This prevents leaking node identity through use + of identical non-default port numbers. (#15651) + +Mempool and transaction relay +----------------------------- + +- Allows one extra single-ancestor transaction per package. Previously, + if a transaction in the mempool had 25 descendants, or it and all of + its descendants were over 101,000 vbytes, any newly-received + transaction that was also a descendant would be ignored. Now, one + extra descendant will be allowed provided it is an immediate + descendant (child) and the child's size is 10,000 vbytes or less. + This makes it possible for two-party contract protocols such as + Lightning Network to give each participant an output they can spend + immediately for Child-Pays-For-Parent (CPFP) fee bumping without + allowing one malicious participant to fill the entire package and thus + prevent the other participant from spending their output. (#15681) + +- Transactions with outputs paying v1 to v16 witness versions (future + segwit versions) are now accepted into the mempool, relayed, and + mined. Attempting to spend those outputs remains forbidden by policy + ("non-standard"). When this change has been widely deployed, wallets + and services can accept any valid bech32 Bitcoin address without + concern that transactions paying future segwit versions will become + stuck in an unconfirmed state. (#15846) + +- Legacy transactions (transactions with no segwit inputs) must now be + sent using the legacy encoding format, enforcing the rule specified in + BIP144. (#14039) + +Wallet +------ + +- When in pruned mode, a rescan that was triggered by an `importwallet`, + `importpubkey`, `importaddress`, or `importprivkey` RPC will only fail + when blocks have been pruned. Previously it would fail when `-prune` + has been set. This change allows setting `-prune` to a high value + (e.g. the disk size) without the calls to any of the import RPCs + failing until the first block is pruned. (#15870) + +- When creating a transaction with a fee above `-maxtxfee` (default 0.1 + BTC), the RPC commands `walletcreatefundedpsbt` and + `fundrawtransaction` will now fail instead of rounding down the fee. + Be aware that the `feeRate` argument is specified in BTC per 1,000 + vbytes, not satoshi per vbyte. (#16257) + +- A new wallet flag `avoid_reuse` has been added (default off). When + enabled, a wallet will distinguish between used and unused addresses, + and default to not use the former in coin selection. When setting + this flag on an existing wallet, rescanning the blockchain is required + to correctly mark previously used destinations. Together with "avoid + partial spends" (added in Bitcoin Core v0.17.0), this can eliminate a + serious privacy issue where a malicious user can track spends by + sending small payments to a previously-paid address that would then + be included with unrelated inputs in future payments. (#13756) + +Build system changes +-------------------- + +- Python >=3.5 is now required by all aspects of the project. This + includes the build systems, test framework and linters. The previously + supported minimum (3.4), was EOL in March 2019. (#14954) + +- The minimum supported miniUPnPc API version is set to 10. This keeps + compatibility with Ubuntu 16.04 LTS and Debian 8 `libminiupnpc-dev` + packages. Please note, on Debian this package is still vulnerable to + [CVE-2017-8798](https://security-tracker.debian.org/tracker/CVE-2017-8798) + (in jessie only) and + [CVE-2017-1000494](https://security-tracker.debian.org/tracker/CVE-2017-1000494) + (both in jessie and in stretch). (#15993) + +0.19.0 change log +================= + +### Consensus +- #16128 Delete error-prone CScript constructor only used with FindAndDelete (instagibbs) +- #16060 Bury bip9 deployments (jnewbery) + +### Policy +- #15557 Enhance `bumpfee` to include inputs when targeting a feerate (instagibbs) +- #15846 Make sending to future native witness outputs standard (sipa) + +### Block and transaction handling +- #15632 Remove ResendWalletTransactions from the Validation Interface (jnewbery) +- #14121 Index for BIP 157 block filters (jimpo) +- #15141 Rewrite DoS interface between validation and net_processing (sdaftuar) +- #15880 utils and libraries: Replace deprecated Boost Filesystem functions (hebasto) +- #15971 validation: Add compile-time checking for negative locking requirement in LimitValidationInterfaceQueue (practicalswift) +- #15999 init: Remove dead code in LoadChainTip (MarcoFalke) +- #16015 validation: Hold cs_main when reading chainActive in RewindBlockIndex (practicalswift) +- #16056 remove unused magic number from consistency check (instagibbs) +- #16171 Remove -mempoolreplacement to prevent needless block prop slowness (TheBlueMatt) +- #15894 Remove duplicated "Error: " prefix in logs (hebasto) +- #14193 validation: Add missing mempool locks (MarcoFalke) +- #15681 Allow one extra single-ancestor transaction per package (TheBlueMatt) +- #15305 [validation] Crash if disconnecting a block fails (sdaftuar) +- #16471 log correct messages when CPFP fails (jnewbery) +- #16433 txmempool: Remove unused default value MemPoolRemovalReason::UNKNOWN (MarcoFalke) +- #13868 Remove unused fScriptChecks parameter from CheckInputs (Empact) +- #16421 Conservatively accept RBF bumps bumping one tx at the package limits (TheBlueMatt) +- #16854 Prevent UpdateTip log message from being broken up (stevenroose) +- #16956 validation: Make GetWitnessCommitmentIndex public (MarcoFalke) +- #16713 Ignore old versionbit activations to avoid 'unknown softforks' warning (jnewbery) +- #17002 chainparams: Bump assumed chain params (MarcoFalke) +- #16849 Fix block index inconsistency in InvalidateBlock() (sdaftuar) + +### P2P protocol and network code +- #15597 Generate log entry when blocks messages are received unexpectedly (pstratem) +- #15654 Remove unused unsanitized user agent string CNode::strSubVer (MarcoFalke) +- #15689 netaddress: Update CNetAddr for ORCHIDv2 (dongcarl) +- #15834 Fix transaction relay bugs introduced in #14897 and expire transactions from peer in-flight map (sdaftuar) +- #15651 torcontrol: Use the default/standard network port for Tor hidden services, even if the internal port is set differently (luke-jr) +- #16188 Document what happens to getdata of unknown type (MarcoFalke) +- #15649 Add ChaCha20Poly1305@Bitcoin AEAD (jonasschnelli) +- #16152 Disable bloom filtering by default (TheBlueMatt) +- #15993 Drop support of the insecure miniUPnPc versions (hebasto) +- #16197 Use mockable time for tx download (MarcoFalke) +- #16248 Make whitebind/whitelist permissions more flexible (NicolasDorier) +- #16618 [Fix] Allow connection of a noban banned peer (NicolasDorier) +- #16631 Restore default whitelistrelay to true (NicolasDorier) +- #15759 Add 2 outbound block-relay-only connections (sdaftuar) +- #15558 Don't query all DNS seeds at once (sipa) +- #16999 0.19 seeds update (laanwj) + +### Wallet +- #15288 Remove wallet -> node global function calls (ryanofsky) +- #15491 Improve log output for errors during load (gwillen) +- #13541 wallet/rpc: sendrawtransaction maxfeerate (kallewoof) +- #15680 Remove resendwallettransactions RPC method (jnewbery) +- #15508 Refactor analyzepsbt for use outside RPC code (gwillen) +- #15747 Remove plethora of Get*Balance (MarcoFalke) +- #15728 Refactor relay transactions (jnewbery) +- #15639 bitcoin-wallet tool: Drop libbitcoin_server.a dependency (ryanofsky) +- #15853 Remove unused import checkpoints.h (MarcoFalke) +- #15780 add cachable amounts for caching credit/debit values (kallewoof) +- #15778 Move maxtxfee from node to wallet (jnewbery) +- #15901 log on rescan completion (andrewtoth) +- #15917 Avoid logging no_such_file_or_directory error (promag) +- #15452 Replace CScriptID and CKeyID in CTxDestination with dedicated types (instagibbs) +- #15870 Only fail rescan when blocks have actually been pruned (MarcoFalke) +- #15006 Add option to create an encrypted wallet (achow101) +- #16001 Give WalletModel::UnlockContext move semantics (sipa) +- #15741 Batch write imported stuff in importmulti (achow101) +- #16144 do not encrypt wallets with disabled private keys (mrwhythat) +- #15024 Allow specific private keys to be derived from descriptor (meshcollider) +- #13756 "avoid_reuse" wallet flag for improved privacy (kallewoof) +- #16226 Move ismine to the wallet module (achow101) +- #16239 wallet/rpc: follow-up clean-up/fixes to avoid_reuse (kallewoof) +- #16286 refactoring: wallet: Fix GCC 7.4.0 warning (hebasto) +- #16257 abort when attempting to fund a transaction above -maxtxfee (Sjors) +- #16237 Have the wallet give out destinations instead of keys (achow101) +- #16322 Fix -maxtxfee check by moving it to CWallet::CreateTransaction (promag) +- #16361 Remove redundant pre-TopUpKeypool check (instagibbs) +- #16244 Move wallet creation out of the createwallet rpc into its own function (achow101) +- #16227 Refactor CWallet's inheritance chain (achow101) +- #16208 Consume ReserveDestination on successful CreateTransaction (instagibbs) +- #16301 Use CWallet::Import* functions in all import* RPCs (achow101) +- #16402 Remove wallet settings from chainparams (MarcoFalke) +- #16415 Get rid of PendingWalletTx class (ryanofsky) +- #15588 Log the actual wallet file version and no longer publicly expose the "version" record (achow101) +- #16399 Improve wallet creation (fjahr) +- #16475 Enumerate walletdb keys (MarcoFalke) +- #15709 Do not add "setting" key as unknown (Bushstar) +- #16451 Remove CMerkleTx (jnewbery) +- #15906 Move min_depth and max_depth to coin control (amitiuttarwar) +- #16502 Drop unused OldKey (promag) +- #16394 Allow createwallet to take empty passwords to make unencrypted wallets (achow101) +- #15911 Use wallet RBF default for walletcreatefundedpsbt (Sjors) +- #16503 Remove p2pEnabled from Chain interface (ariard) +- #16557 restore coinbase and confirmed/conflicted checks in SubmitMemoryPoolAndRelay() (jnewbery) +- #14934 Descriptor expansion cache clarifications (Sjors) +- #16383 rpcwallet: default include_watchonly to true for watchonly wallets (jb55) +- #16542 Return more specific errors about invalid descriptors (achow101) +- #16572 Fix Char as Bool in Wallet (JeremyRubin) +- #16753 extract PubKey from P2PK script with Solver (theStack) +- #16716 Use wallet name instead of pointer on unload/release (promag) +- #16185 gettransaction: add an argument to decode the transaction (darosior) +- #16745 Translate all initErrors in CreateWalletFromFile (MarcoFalke) +- #16792 Assert that the HRP is lowercase in Bech32::Encode (meshcollider) +- #16624 encapsulate transactions state (ariard) +- #16830 Cleanup walletinitinterface.h (hebasto) +- #16796 Fix segfault in CreateWalletFromFile (MarcoFalke) +- #16866 Rename 'decode' argument in gettransaction method to 'verbose' (jnewbery) +- #16727 Explicit feerate for bumpfee (instagibbs) +- #16609 descriptor: fix missed m_script_arg arg renaming in #14934 (fanquake) + +### RPC and other APIs +- #15492 remove deprecated generate method (Sjors) +- #15566 cli: Replace testnet with chain and return network name as per bip70 (fanquake) +- #15564 cli: Remove duplicate wallet fields from -getinfo (fanquake) +- #15642 Remove deprecated rpc warnings (jnewbery) +- #15637 Rename size to vsize in mempool related calls (fanquake) +- #15620 Uncouple non-wallet rpcs from maxTxFee global (MarcoFalke) +- #15616 Clarify decodescript RPCResult doc (MarcoFalke) +- #15669 Fix help text for signtransactionwithXXX (torkelrogstad) +- #15596 Ignore sendmany::minconf as dummy value (MarcoFalke) +- #15755 remove unused var in rawtransaction.cpp (Bushstar) +- #15746 RPCHelpMan: Always name dictionary keys (MarcoFalke) +- #15748 remove dead mining code (jnewbery) +- #15751 Speed up deriveaddresses for large ranges (sipa) +- #15770 Validate maxfeerate with AmountFromValue (promag) +- #15474 rest/rpc: Make mempoolinfo atomic (promag) +- #15463 Speedup getaddressesbylabel (promag) +- #15784 Remove dependency on interfaces::Chain in SignTransaction (ariard) +- #15323 Expose g_is_mempool_loaded via getmempoolinfo (Empact) +- #15932 Serialize in getblock without cs_main (MarcoFalke) +- #15930 Add balances RPC (MarcoFalke) +- #15730 Show scanning details in getwalletinfo (promag) +- #14802 faster getblockstats using BlockUndo data (FelixWeis) +- #14984 Speedup getrawmempool when verbose=true (promag) +- #16071 Hint for importmulti in help output of importpubkey and importaddress (kristapsk) +- #16063 Mention getwalletinfo where a rescan is triggered (promag) +- #16024 deriveaddresses: Correction of descriptor checksum in RPC example (ccapo) +- #16217 getrawtransaction: inform about blockhash argument when lookup fails (darosior) +- #15427 Add support for descriptors to utxoupdatepsbt (sipa) +- #16262 Allow shutdown while in generateblocks (pstratem) +- #15483 Adding a 'logpath' entry to getrpcinfo (darosior) +- #16325 Clarify that block count means height excl genesis (MarcoFalke) +- #16326 add new utxoupdatepsbt arguments to the CRPCCommand and CPRCCvertParam tables (jnewbery) +- #16332 Add logpath description for getrpcinfo (instagibbs) +- #16240 JSONRPCRequest-aware RPCHelpMan (kallewoof) +- #15996 Deprecate totalfee argument in `bumpfee` (instagibbs) +- #16467 sendrawtransaction help privacy note (jonatack) +- #16596 Fix getblocktemplate CLI example (emilengler) +- #15986 Add checksum to getdescriptorinfo (sipa) +- #16647 add weight to getmempoolentry output (fanquake) +- #16695 Add window final block height to getchaintxstats (leto) +- #16798 Refactor rawtransaction_util's SignTransaction to separate prevtx parsing (achow101) +- #16285 Improve scantxoutset response and help message (promag) +- #16725 Don't show addresses or P2PK in decoderawtransaction (NicolasDorier) +- #16787 Human readable network services (darosior) +- #16251 Improve signrawtransaction error reporting (ajtowns) +- #16873 fix regression in gettransaction (jonatack) +- #16512 Shuffle inputs and outputs after joining psbts (achow101) +- #16521 Use the default maxfeerate value as BTC/kB (Remagpie) +- #16817 Fix casing in getblockchaininfo to be inline with other fields (dangershony) +- #17131 fix -rpcclienttimeout 0 option (fjahr) +- #17249 Add missing deque include to fix build (jbeich) + +### GUI +- #15464 Drop unused return values in WalletFrame (promag) +- #15614 Defer removeAndDeleteWallet when no modal widget is active (promag) +- #15711 Generate bech32 addresses by default (MarcoFalke) +- #15829 update request payment button text and tab description (fanquake) +- #15874 Resolve the qt/guiutil <-> qt/optionsmodel CD (251Labs) +- #15371 Uppercase bech32 addresses in qr codes (benthecarman) +- #15928 Move QRImageWidget to its own file-pair (luke-jr) +- #16113 move coin control "OK" to the right hand side of the dialog (fanquake) +- #16090 Add vertical spacer to peer detail widget (JosuGZ) +- #15886 qt, wallet: Revamp SendConfirmationDialog (hebasto) +- #16263 Use qInfo() if no error occurs (hebasto) +- #16153 Add antialiasing to traffic graph widget (JosuGZ) +- #16350 Remove unused guard (hebasto) +- #16106 Sort wallets in open wallet menu (promag) +- #16291 Stop translating PACKAGE_NAME (MarcoFalke) +- #16380 Remove unused bits from the service flags enum (MarcoFalke) +- #16379 Fix autostart filenames on Linux for testnet/regtest (hebasto) +- #16366 init: Use InitError for all errors in bitcoind/qt (MarcoFalke) +- #16436 Do not create payment server if -disablewallet option provided (hebasto) +- #16514 Remove unused RPCConsole::tabFocus (promag) +- #16497 Generate bech32 addresses by default (take 2, fixup) (MarcoFalke) +- #16349 Remove redundant WalletController::addWallet slot (hebasto) +- #16578 Do not pass in command line arguments to QApplication (achow101) +- #16612 Remove menu icons (laanwj) +- #16677 remove unused PlatformStyle::TextColorIcon (fanquake) +- #16694 Ensure transaction send error is always visible (fanquake) +- #14879 Add warning messages to the debug window (hebasto) +- #16708 Replace obsolete functions of QSslSocket (hebasto) +- #16701 Replace functions deprecated in Qt 5.13 (hebasto) +- #16706 Replace deprecated QSignalMapper by lambda expressions (hebasto) +- #16707 Remove obsolete QModelIndex::child() (hebasto) +- #16758 Replace QFontMetrics::width() with TextWidth() (hebasto) +- #16760 Change uninstall icon on Windows (GChuf) +- #16720 Replace objc_msgSend() function calls with the native Objective-C syntax (hebasto) +- #16788 Update transifex slug for 0.19 (laanwj) +- #15450 Create wallet menu option (achow101) +- #16735 Remove unused menu items for Windows and Linux (GChuf) +- #16826 Do additional character escaping for wallet names and address labels (achow101) +- #15529 Add Qt programs to msvc build (updated, no code changes) (sipsorcery) +- #16714 add prune to intro screen with smart default (Sjors) +- #16858 advise users not to switch wallets when opening a BIP70 URI (jameshilliard) +- #16822 Create wallet menu option follow-ups (jonatack) +- #16882 Re-generate translations before 0.19.0 (MarcoFalke) +- #16928 Rename address checkbox back to bech32 (MarcoFalke) +- #16837 Fix {C{,XX},LD}FLAGS pickup (dongcarl) +- #16971 Change default size of intro frame (emilengler) +- #16988 Periodic translations update (laanwj) +- #16852 When BIP70 is disabled, get PaymentRequest merchant using string search (achow101) +- #16952 make sure to update the UI when deleting a transaction (jonasschnelli) +- #17031 Prevent processing duplicate payment requests (promag) +- #17135 Make polling in ClientModel asynchronous (promag) +- #17120 Fix start timer from non QThread (promag) +- #17257 disable font antialiasing for QR image address (fanquake) + +### Build system +- #14954 Require python 3.5 (MarcoFalke) +- #15580 native_protobuf: avoid system zlib (dongcarl) +- #15601 Switch to python3 (take 3) (MarcoFalke) +- #15581 Make less assumptions about build env (dongcarl) +- #14853 latest RapidCheck (fanquake) +- #15446 Improve depends debuggability (dongcarl) +- #13788 Fix --disable-asm for newer assembly checks/code (luke-jr) +- #12051 add missing debian contrib file to tarball (puchu) +- #15919 Remove unused OpenSSL includes to make it more clear where OpenSSL is used (practicalswift) +- #15978 .gitignore: Don't ignore depends patches (dongcarl) +- #15939 gitian: Remove windows 32 bit build (MarcoFalke) +- #15239 scripts and tools: Move non-linux build source tarballs to "bitcoin-binaries/version" directory (hebasto) +- #14047 Add HKDF_HMAC256_L32 and method to negate a private key (jonasschnelli) +- #16051 add patch to common dependencies (fanquake) +- #16049 switch to secure download of all dependencies (Kemu) +- #16059 configure: Fix thread_local detection (dongcarl) +- #16089 add ability to skip building zeromq (fanquake) +- #15844 Purge libtool archives (dongcarl) +- #15461 update to Boost 1.70 (Sjors) +- #16141 remove GZIP export from gitian descriptors (fanquake) +- #16235 Cleaned up and consolidated msbuild files (no code changes) (sipsorcery) +- #16246 MSVC: Fix error in debug mode (Fix #16245) (NicolasDorier) +- #16183 xtrans: Configure flags cleanup (dongcarl) +- #16258 [MSVC]: Create the config.ini as part of bitcoind build (NicolasDorier) +- #16271 remove -Wall from rapidcheck build flags (fanquake) +- #16309 [MSVC] allow user level project customization (NicolasDorier) +- #16308 [MSVC] Copy build output to src/ automatically after build (NicolasDorier) +- #15457 Check std::system for -[alert|block|wallet]notify (Sjors) +- #16344 use #if HAVE_SYSTEM instead of defined(HAVE_SYSTEM) (Sjors) +- #16352 prune dbus from depends (fanquake) +- #16270 expat 2.2.7 (fanquake) +- #16408 Prune X packages (dongcarl) +- #16386 disable unused Qt features (fanquake) +- #16424 Treat -Wswitch as error when --enable-werror (MarcoFalke) +- #16441 remove qt libjpeg check from bitcoin_qt.m4 (fanquake) +- #16434 Specify AM_CPPFLAGS for ZMQ (domob1812) +- #16534 add Qt Creator Makefile.am.user to .gitignore (Bushstar) +- #16573 disable building libsecp256k1 benchmarks (fanquake) +- #16533 disable libxcb extensions (fanquake) +- #16589 Remove unused src/obj-test folder (MarcoFalke) +- #16435 autoconf: Sane `--enable-debug` defaults (dongcarl) +- #16622 echo property tests status during build (jonatack) +- #16611 Remove src/obj directory from repository (laanwj) +- #16371 ignore macOS make deploy artefacts & add them to clean-local (fanquake) +- #16654 build: update RapidCheck Makefile (jonatack) +- #16370 cleanup package configure flags (fanquake) +- #16746 msbuild: Ignore linker warning (sipsorcery) +- #16750 msbuild: adds bench_bitcoin to auto generated project files (sipsorcery) +- #16810 guix: Remove ssp spec file hack (dongcarl) +- #16477 skip deploying plugins we dont use in macdeployqtplus (fanquake) +- #16413 Bump QT to LTS release 5.9.8 (THETCR) +- #15584 disable BIP70 support by default (fanquake) +- #16871 make building protobuf optional in depends (fanquake) +- #16879 remove redundant sed patching (fanquake) +- #16809 zlib: Move toolchain options to configure (dongcarl) +- #15146 Solve SmartOS FD_ZERO build issue (Empact) +- #16870 update boost macros to latest upstream for improved error reporting (fanquake) +- #16982 Factor out qt translations from build system (laanwj) +- #16926 Add OpenSSL termios fix for musl libc (nmarley) +- #16927 Refresh ZeroMQ 4.3.1 patch (nmarley) +- #17005 Qt version appears only if GUI is being built (ch4ot1c) +- #16468 Exclude depends/Makefile in .gitignore (promag) + +### Tests and QA +- #15296 Add script checking for deterministic line coverage in unit tests (practicalswift) +- #15338 ci: Build and run tests once on freebsd (MarcoFalke) +- #15479 Add .style.yapf (MarcoFalke) +- #15534 lint-format-strings: open files sequentially (fix for OS X) (gwillen) +- #15504 fuzz: Link BasicTestingSetup (shared with unit tests) (MarcoFalke) +- #15473 bench: Benchmark mempooltojson (MarcoFalke) +- #15466 Print remaining jobs in test_runner.py (stevenroose) +- #15631 mininode: Clearer error message on invalid magic bytes (MarcoFalke) +- #15255 Remove travis_wait from lint script (gkrizek) +- #15686 make pruning test faster (jnewbery) +- #15533 .style.yapf: Set column_limit=160 (MarcoFalke) +- #15660 Overhaul p2p_compactblocks.py (sdaftuar) +- #15495 Add regtests for HTTP status codes (domob1812) +- #15772 Properly log named args in authproxy (MarcoFalke) +- #15771 Prevent concurrency issues reading .cookie file (promag) +- #15693 travis: Switch to ubuntu keyserver to avoid timeouts (MarcoFalke) +- #15629 init: Throw error when network specific config is ignored (MarcoFalke) +- #15773 Add BitcoinTestFramework::sync_* methods (MarcoFalke) +- #15797 travis: Bump second timeout to 33 minutes, add rationale (MarcoFalke) +- #15788 Unify testing setups for fuzz, bench, and unit tests (MarcoFalke) +- #15352 Reduce noise level in test_bitcoin output (practicalswift) +- #15779 Add wallet_balance benchmark (MarcoFalke) +- #15843 fix outdated include in blockfilter_index_tests (jamesob) +- #15866 Add missing syncwithvalidationinterfacequeue to wallet_import_rescan (MarcoFalke) +- #15697 Make swap_magic_bytes in p2p_invalid_messages atomic (MarcoFalke) +- #15895 Avoid re-reading config.ini unnecessarily (luke-jr) +- #15896 feature_filelock, interface_bitcoin_cli: Use PACKAGE_NAME in messages rather than hardcoding Bitcoin Core (luke-jr) +- #15897 QA/mininode: Send all headers upfront in send_blocks_and_test to avoid sending an unconnected one (luke-jr) +- #15696 test_runner: Move feature_pruning to base tests (MarcoFalke) +- #15869 Add settings merge test to prevent regresssions (ryanofsky) +- #15758 Add further tests to wallet_balance (MarcoFalke) +- #15841 combine_logs: append node stderr and stdout if it exists (MarcoFalke) +- #15949 test_runner: Move pruning back to extended (MarcoFalke) +- #15927 log thread names by default in functional tests (jnewbery) +- #15664 change default Python block serialization to witness (instagibbs) +- #15988 Add test for ArgsManager::GetChainName (ryanofsky) +- #15963 Make random seed logged and settable (jnewbery) +- #15943 Fail if RPC has been added without tests (MarcoFalke) +- #16036 travis: Run all lint scripts even if one fails (scravy) +- #13555 parameterize adjustment period in versionbits_computeblockversion (JBaczuk) +- #16079 wallet_balance.py: Prevent edge cases (stevenroose) +- #16078 replace tx hash with txid in rawtransaction test (LongShao007) +- #16042 Bump MAX_NODES to 12 (MarcoFalke) +- #16124 Limit Python linting to files in the repo (practicalswift) +- #16143 Mark unit test blockfilter_index_initial_sync as non-deterministic (practicalswift) +- #16214 travis: Fix caching issues (MarcoFalke) +- #15982 Make msg_block a witness block (MarcoFalke) +- #16225 Make coins_tests/updatecoins_simulation_test deterministic (practicalswift) +- #16236 fuzz: Log output even if fuzzer failed (MarcoFalke) +- #15520 cirrus: Run extended test feature_pruning (MarcoFalke) +- #16234 Add test for unknown args (MarcoFalke) +- #16207 stop generating lcov coverage when functional tests fail (asood123) +- #16252 Log to debug.log in all unit tests (MarcoFalke) +- #16289 Add missing ECC_Stop() in GUI rpcnestedtests.cpp (jonasschnelli) +- #16278 Remove unused includes (practicalswift) +- #16302 Add missing syncwithvalidationinterfacequeue to wallet_balance test (MarcoFalke) +- #15538 wallet_bumpfee.py: Make sure coin selection produces change (instagibbs) +- #16294 Create at most one testing setup (MarcoFalke) +- #16299 bench: Move generated data to a dedicated translation unit (promag) +- #16329 Add tests for getblockchaininfo.softforks (MarcoFalke) +- #15687 tool wallet test coverage for unexpected writes to wallet (jonatack) +- #16267 bench: Benchmark blocktojson (fanatid) +- #14505 Add linter to make sure single parameter constructors are marked explicit (practicalswift) +- #16338 Disable other targets when enable-fuzz is set (qmma70) +- #16334 rpc_users: Also test rpcauth.py with password (dongcarl) +- #15282 Replace hard-coded hex tx with class in test framework (stevenroose) +- #16390 Add --filter option to test_runner.py (promag) +- #15891 Require standard txs in regtest by default (MarcoFalke) +- #16374 Enable passing wildcard test names to test runner from root (jonatack) +- #16420 Fix race condition in wallet_encryption test (jonasschnelli) +- #16422 remove redundant setup in addrman_tests (zenosage) +- #16438 travis: Print memory and number of cpus (MarcoFalke) +- #16445 Skip flaky p2p_invalid_messages test on macOS (fjahr) +- #16459 Fix race condition in example_test.py (sdaftuar) +- #16464 Ensure we don't generate a too-big block in p2sh sigops test (sdaftuar) +- #16491 fix deprecated log.warn in feature_dbcrash test (jonatack) +- #15134 Switch one of the Travis jobs to an unsigned char environment (-funsigned-char) (practicalswift) +- #16505 Changes verbosity of msbuild from quiet to normal in the appveyor script (sipsorcery) +- #16293 Make test cases separate functions (MarcoFalke) +- #16470 Fail early on disconnect in mininode.wait_for_* (MarcoFalke) +- #16277 Suppress output in test_bitcoin for expected errors (gertjaap) +- #16493 Fix test failures (MarcoFalke) +- #16538 Add missing sync_blocks to feature_pruning (MarcoFalke) +- #16509 Adapt test framework for chains other than "regtest" (MarcoFalke) +- #16363 Add test for BIP30 duplicate tx (MarcoFalke) +- #16535 Explain why -whitelist is used in feature_fee_estimation (MarcoFalke) +- #16554 only include and use OpenSSL where it's actually needed (BIP70) (fanquake) +- #16598 Remove confusing hash256 function in util (elichai) +- #16595 travis: Use extended 90 minute timeout when available (MarcoFalke) +- #16563 Add unit test for AddTimeData (mzumsande) +- #16561 Use colors and dots in test_runner.py output only if standard output is a terminal (practicalswift) +- #16465 Test p2sh-witness and bech32 in wallet_import_rescan (MarcoFalke) +- #16582 Rework ci (Use travis only as fallback env) (MarcoFalke) +- #16633 travis: Fix test_runner.py timeouts (MarcoFalke) +- #16646 Run tests with UPnP disabled (fanquake) +- #16623 ci: Add environment files for all settings (MarcoFalke) +- #16656 fix rpc_setban.py race (jonasschnelli) +- #16570 Make descriptor tests deterministic (davereikher) +- #16404 Test ZMQ notification after chain reorg (promag) +- #16726 Avoid common Python default parameter gotcha when mutable dict/list:s are used as default parameter values (practicalswift) +- #16739 ci: Pass down $makejobs to test_runner.py, other improvements (MarcoFalke) +- #16767 Check for codespell in lint-spelling.sh (kristapsk) +- #16768 Make lint-includes.sh work from any directory (kristapsk) +- #15257 Scripts and tools: Bump flake8 to 3.7.8 (Empact) +- #16804 Remove unused try-block in assert_debug_log (MarcoFalke) +- #16850 `servicesnames` field in `getpeerinfo` and `getnetworkinfo` (darosior) +- #16551 Test that low difficulty chain fork is rejected (MarcoFalke) +- #16737 Establish only one connection between nodes in rpc_invalidateblock (MarcoFalke) +- #16845 Add notes on how to generate data/wallets/high_minversion (MarcoFalke) +- #16888 Bump timeouts in slow running tests (MarcoFalke) +- #16864 Add python bech32 impl round-trip test (instagibbs) +- #16865 add some unit tests for merkle.cpp (soroosh-sdi) +- #14696 Add explicit references to related CVE's in p2p_invalid_block test (lucash-dev) +- #16907 lint: Add DisabledOpcodeTemplates to whitelist (MarcoFalke) +- #16898 Remove connect_nodes_bi (MarcoFalke) +- #16917 Move common function assert_approx() into util.py (fridokus) +- #16921 Add information on how to add Vulture suppressions (practicalswift) +- #16920 Fix extra_args in wallet_import_rescan.py (MarcoFalke) +- #16918 Make PORT_MIN in test runner configurable (MarcoFalke) +- #16941 travis: Disable feature_block in tsan run due to oom (MarcoFalke) +- #16929 follow-up to rpc: default maxfeerate value as BTC/kB (jonatack) +- #16959 ci: Set $host before setting fallback values (MarcoFalke) +- #16961 Remove python dead code linter (laanwj) +- #16931 add unittests for CheckProofOfWork (soroosh-sdi) +- #16991 Fix service flag comparison check in rpc_net test (luke-jr) (laanwj) +- #16987 Correct docstring param name (jbampton) +- #17015 Explain QT_QPA_PLATFORM for gui tests (MarcoFalke) +- #17006 Enable UBSan for Travis fuzzing job (practicalswift) +- #17086 Fix fs_tests for unknown locales (carnhofdaki) +- #15903 appveyor: Write @PACKAGE_NAME@ to config (MarcoFalke) +- #16742 test: add executable flag for wallet_watchonly.py (theStack) +- #16740 qa: Relax so that the subscriber is ready before publishing zmq messages (#16740) + +### Miscellaneous +- #15335 Fix lack of warning of unrecognized section names (AkioNak) +- #15528 contrib: Bump gitian descriptors for 0.19 (MarcoFalke) +- #15609 scripts and tools: Set 'distro' explicitly (hebasto) +- #15519 Add Poly1305 implementation (jonasschnelli) +- #15643 contrib: Gh-merge: include acks in merge commit (MarcoFalke) +- #15838 scripts and tools: Fetch missing review comments in github-merge.py (nkostoulas) +- #15920 lint: Check that all wallet args are hidden (MarcoFalke) +- #15849 Thread names in logs and deadlock debug tools (jamesob) +- #15650 Handle the result of posix_fallocate system call (lucayepa) +- #15766 scripts and tools: Upgrade gitian image before signing (hebasto) +- #15512 Add ChaCha20 encryption option (XOR) (jonasschnelli) +- #15968 Fix portability issue with pthreads (grim-trigger) +- #15970 Utils and libraries: fix static_assert for macro HAVE_THREAD_LOCAL (orientye) +- #15863 scripts and tools: Ensure repos are up-to-date in gitian-build.py (hebasto) +- #15224 Add RNG strengthening (10ms once every minute) (sipa) +- #15840 Contrib scripts: Filter IPv6 by ASN (abitfan) +- #13998 Scripts and tools: gitian-build.py improvements and corrections (hebasto) +- #15236 scripts and tools: Make --setup command independent (hebasto) +- #16114 contrib: Add curl as a required program in gitian-build.py (fanquake) +- #16046 util: Add type safe gettime (MarcoFalke) +- #15703 Update secp256k1 subtree to latest upstream (sipa) +- #16086 contrib: Use newer config.guess & config.sub in install_db4.sh (fanquake) +- #16130 Don't GPG sign intermediate commits with github-merge tool (stevenroose) +- #16162 scripts: Add key for michael ford (fanquake) to trusted keys list (fanquake) +- #16201 devtools: Always use unabbreviated commit IDs in github-merge.py (laanwj) +- #16112 util: Log early messages (MarcoFalke) +- #16223 devtools: Fetch and display ACKs at sign-off time in github-merge (laanwj) +- #16300 util: Explain why the path is cached (MarcoFalke) +- #16314 scripts and tools: Update copyright_header.py script (hebasto) +- #16158 Fix logic of memory_cleanse() on MSVC and clean up docs (real-or-random) +- #14734 fix an undefined behavior in uint::SetHex (kazcw) +- #16327 scripts and tools: Update ShellCheck linter (hebasto) +- #15277 contrib: Enable building in guix containers (dongcarl) +- #16362 Add bilingual_str type (hebasto) +- #16481 logs: add missing space (harding) +- #16581 sipsorcery gitian key (sipsorcery) +- #16566 util: Refactor upper/lowercase functions (kallewoof) +- #16620 util: Move resolveerrmsg to util/error (MarcoFalke) +- #16625 scripts: Remove github-merge.py (fanquake) +- #15864 Fix datadir handling (hebasto) +- #16670 util: Add join helper to join a list of strings (MarcoFalke) +- #16665 scripts: Move update-translations.py to maintainer-tools repo (fanquake) +- #16730 Support serialization of `std::vector<bool>` (sipa) +- #16556 Fix systemd service file configuration directory setup (setpill) +- #15615 Add log output during initial header sync (jonasschnelli) +- #16774 Avoid unnecessary "Synchronizing blockheaders" log messages (jonasschnelli) +- #16489 log: harmonize bitcoind logging (jonatack) +- #16577 util: Cbufferedfile fixes and unit test (LarryRuane) +- #16984 util: Make thread names shorter (hebasto) +- #17038 Don't rename main thread at process level (laanwj) +- #17184 util: Filter out macos process serial number (hebasto) +- #17095 util: Filter control characters out of log messages (laanwj) +- #17085 init: Change fallback locale to C.UTF-8 (laanwj) +- #16957 9% less memory: make SaltedOutpointHasher noexcept (martinus) + +### Documentation +- #15514 Update Transifex links (fanquake) +- #15513 add "sections" info to example bitcoin.conf (fanquake) +- #15530 Move wallet lock annotations to header (MarcoFalke) +- #15562 remove duplicate clone step in build-windows.md (fanquake) +- #15565 remove release note fragments (fanquake) +- #15444 Additional productivity tips (Sjors) +- #15577 Enable TLS in link to chris.beams.io (JeremyRand) +- #15604 release note for disabling reject messages by default (jnewbery) +- #15611 Add Gitian key for droark (droark) +- #15626 Update ACK description in CONTRIBUTING.md (jonatack) +- #15603 Add more tips to productivity.md (gwillen) +- #15683 Comment for seemingly duplicate LIBBITCOIN_SERVER (Bushstar) +- #15685 rpc-mining: Clarify error messages (MarcoFalke) +- #15760 Clarify sendrawtransaction::maxfeerate==0 help (MarcoFalke) +- #15659 fix findFork comment (r8921039) +- #15718 Improve netaddress comments (dongcarl) +- #15833 remove out-of-date comment on pay-to-witness support (r8921039) +- #15821 Remove upgrade note in release notes from EOL versions (MarcoFalke) +- #15267 explain AcceptToMemoryPoolWorker's coins_to_uncache (jamesob) +- #15887 Align code example style with clang-format (hebasto) +- #15877 Fix -dustrelayfee= argument docs grammar (keepkeyjon) +- #15908 Align MSVC build options with Linux build ones (hebasto) +- #15941 Add historical release notes for 0.18.0 (laanwj) +- #15794 Clarify PR guidelines w/re documentation (dongcarl) +- #15607 Release process updates (jonatack) +- #14364 Clarify -blocksdir usage (sangaman) +- #15777 Add doxygen comments for keypool classes (jnewbery) +- #15820 Add productivity notes for dummy rebases (dongcarl) +- #15922 Explain how to pass in non-fundamental types into functions (MarcoFalke) +- #16080 build/doc: update bitcoin_config.h packages, release process (jonatack) +- #16047 analyzepsbt description in doc/psbt.md (jonatack) +- #16039 add release note for 14954 (fanquake) +- #16139 Add riscv64 to outputs list in release-process.md (JeremyRand) +- #16140 create security policy (narula) +- #16164 update release process for SECURITY.md (jonatack) +- #16213 Remove explicit mention of versions from SECURITY.md (MarcoFalke) +- #16186 doc/lint: Fix spelling errors identified by codespell 1.15.0 (Empact) +- #16149 Rework section on ACK in CONTRIBUTING.md (MarcoFalke) +- #16196 Add release notes for 14897 & 15834 (MarcoFalke) +- #16241 add rapidcheck to vcpkg install list (fanquake) +- #16243 Remove travis badge from readme (MarcoFalke) +- #16256 remove orphaned header in developer notes (jonatack) +- #15964 Improve build-osx document formatting (giulio92) +- #16313 Fix broken link in doc/build-osx.md (jonatack) +- #16330 Use placeholder instead of key expiration date (hebasto) +- #16339 add reduce-memory.md (fanquake) +- #16347 Include static members in Doxygen (dongcarl) +- #15824 Improve netbase comments (dongcarl) +- #16430 Update bips 35, 37 and 111 status (MarcoFalke) +- #16455 Remove downgrading warning in release notes, per 0.18 branch (MarcoFalke) +- #16484 update labels in CONTRIBUTING.md (MarcoFalke) +- #16483 update Python command in msvc readme (sipsorcery) +- #16504 Add release note for the deprecated totalFee option of bumpfee (promag) +- #16448 add note on precedence of options in bitcoin.conf (fanquake) +- #16536 Update and extend benchmarking.md (ariard) +- #16530 Fix grammar and punctuation in developer notes (Tech1k) +- #16574 Add historical release notes for 0.18.1 (laanwj) +- #16585 Update Markdown syntax for bdb packages (emilengler) +- #16586 Mention other ways to conserve memory on compilation (MarcoFalke) +- #16605 Add missing contributor to 0.18.1 release notes (meshcollider) +- #16615 Fix typos in COPYRIGHT (gapeman) +- #16626 Fix spelling error chache -> cache (nilswloewen) +- #16587 Improve versionbits.h documentation (ariard) +- #16643 Add ZMQ dependencies to the Fedora build instructions (hebasto) +- #16634 Refer in rpcbind doc to the manpage (MarcoFalke) +- #16555 mention whitelist is inbound, and applies to blocksonly (Sjors) +- #16645 initial RapidCheck property-based testing documentation (jonatack) +- #16691 improve depends prefix documentation (fanquake) +- #16629 Add documentation for the new whitelist permissions (NicolasDorier) +- #16723 Update labels in CONTRIBUTING.md (hebasto) +- #16461 Tidy up shadowing section (promag) +- #16621 add default bitcoin.conf locations (GChuf) +- #16752 Delete stale URL in test README (michaelfolkson) +- #14862 Declare BLOCK_VALID_HEADER reserved (MarcoFalke) +- #16806 Add issue templates for bug and feature request (MarcoFalke) +- #16857 Elaborate need to re-login on Debian-based after usermod for Tor group (clashicly) +- #16863 Add a missing closing parenthesis in the bitcoin-wallet's help (darosior) +- #16757 CChainState return values (MarcoFalke) +- #16847 add comments clarifying how local services are advertised (jamesob) +- #16812 Fix whitespace errs in .md files, bitcoin.conf, and Info.plist.in (ch4ot1c) +- #16885 Update tx-size-small comment with relevant CVE disclosure (instagibbs) +- #16900 Fix doxygen comment for SignTransaction in rpc/rawtransaction_util (MarcoFalke) +- #16914 Update homebrew instruction for doxygen (Sjors) +- #16912 Remove Doxygen intro from src/bitcoind.cpp (ch4ot1c) +- #16960 replace outdated OpenSSL comment in test README (fanquake) +- #16968 Remove MSVC update step from translation process (laanwj) +- #16953 Improve test READMEs (fjahr) +- #16962 Put PR template in comments (laanwj) +- #16397 Clarify includeWatching for fundrawtransaction (stevenroose) +- #15459 add how to calculate blockchain and chainstate size variables to release process (marcoagner) +- #16997 Update bips.md for 0.19 (laanwj) +- #17001 Remove mention of renamed mapBlocksUnlinked (MarcoFalke) +- #17014 Consolidate release notes before 0.19.0 (move-only) (MarcoFalke) +- #17111 update bips.md with buried BIP9 deployments (MarcoFalke) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 251 +- Aaron Clauson +- Akio Nakamura +- Alistair Mann +- Amiti Uttarwar +- Andrew Chow +- andrewtoth +- Anthony Towns +- Antoine Riard +- Aseem Sood +- Ben Carman +- Ben Woosley +- bpay +- Carl Dong +- Carnhof Daki +- Chris Capobianco +- Chris Moore +- Chuf +- clashic +- clashicly +- Cory Fields +- Daki Carnhof +- Dan Gershony +- Daniel Edgecumbe +- Daniel Kraft +- Daniel McNally +- darosior +- David A. Harding +- David Reikher +- Douglas Roark +- Elichai Turkel +- Emil +- Emil Engler +- ezegom +- Fabian Jahr +- fanquake +- Felix Weis +- Ferdinando M. Ametrano +- fridokus +- gapeman +- GChuf +- Gert-Jaap Glasbergen +- Giulio Lombardo +- Glenn Willen +- Graham Krizek +- Gregory Sanders +- grim-trigger +- gwillen +- Hennadii Stepanov +- Jack Mallers +- James Hilliard +- James O'Beirne +- Jan Beich +- Jeremy Rubin +- JeremyRand +- Jim Posen +- John Bampton +- John Newbery +- Jon Atack +- Jon Layton +- Jonas Schnelli +- Jonathan "Duke" Leto +- João Barbosa +- Joonmo Yang +- Jordan Baczuk +- Jorge Timón +- Josu Goñi +- Julian Fleischer +- Karl-Johan Alm +- Kaz Wesley +- keepkeyjon +- Kirill Fomichev +- Kristaps Kaupe +- Kristian Kramer +- Larry Ruane +- Lenny Maiorani +- LongShao007 +- Luca Venturini +- lucash-dev +- Luke Dashjr +- marcoagner +- MarcoFalke +- marcuswin +- Martin Ankerl +- Martin Zumsande +- Matt Corallo +- MeshCollider +- Michael Folkson +- Miguel Herranz +- Nathan Marley +- Neha Narula +- nicolas.dorier +- Nils Loewen +- nkostoulas +- orient +- Patrick Strateman +- Peter Bushnell +- Peter Wagner +- Pieter Wuille +- practicalswift +- qmma +- r8921039 +- RJ Rybarczyk +- Russell Yanofsky +- Samuel Dobson +- Sebastian Falbesoner +- setpill +- shannon1916 +- Sjors Provoost +- soroosh-sdi +- Steven Roose +- Suhas Daftuar +- tecnovert +- THETCR +- Tim Ruffing +- Tobias Kaderle +- Torkel Rogstad +- Ulrich Kempken +- whythat +- William Casarin +- Wladimir J. van der Laan +- zenosage + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 477985bb33..ffa001bb7f 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -54,43 +54,6 @@ </dict> </array> - <key>UTExportedTypeDeclarations</key> - <array> - <dict> - <key>UTTypeIdentifier</key> - <string>org.bitcoin.paymentrequest</string> - <key>UTTypeDescription</key> - <string>Bitcoin payment request</string> - <key>UTTypeConformsTo</key> - <array> - <string>public.data</string> - </array> - <key>UTTypeTagSpecification</key> - <dict> - <key>public.mime-type</key> - <string>application/x-bitcoin-payment-request</string> - <key>public.filename-extension</key> - <array> - <string>bitcoinpaymentrequest</string> - </array> - </dict> - </dict> - </array> - - <key>CFBundleDocumentTypes</key> - <array> - <dict> - <key>CFBundleTypeRole</key> - <string>Editor</string> - <key>LSItemContentTypes</key> - <array> - <string>org.bitcoin.paymentrequest</string> - </array> - <key>LSHandlerRank</key> - <string>Owner</string> - </dict> - </array> - <key>NSPrincipalClass</key> <string>NSApplication</string> diff --git a/share/setup.nsi.in b/share/setup.nsi.in index e9aa1f2b73..649483c732 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -48,7 +48,7 @@ Var StartMenuGroup !insertmacro MUI_LANGUAGE English # Installer attributes -OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win@WINDOWS_BITS@-setup.exe +OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win@WINDOWS_BITS@-setup-unsigned.exe !if "@WINDOWS_BITS@" == "64" InstallDir $PROGRAMFILES64\Bitcoin !else diff --git a/src/Makefile.am b/src/Makefile.am index d50524a8ae..cbe5479956 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,7 +19,7 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) @@ -119,6 +119,7 @@ BITCOIN_CORE_H = \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ + compat/cpuid.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ @@ -146,19 +147,23 @@ BITCOIN_CORE_H = \ dbwrapper.h \ limitedmap.h \ logging.h \ + logging/timer.h \ memusage.h \ merkleblock.h \ miner.h \ net.h \ net_permissions.h \ net_processing.h \ + net_types.h \ netaddress.h \ netbase.h \ netmessagemaker.h \ node/coin.h \ node/coinstats.h \ + node/context.h \ node/psbt.h \ node/transaction.h \ + node/utxo_snapshot.h \ noui.h \ optional.h \ outputtype.h \ @@ -171,6 +176,7 @@ BITCOIN_CORE_H = \ protocol.h \ psbt.h \ random.h \ + randomenv.h \ reverse_iterator.h \ reverselock.h \ rpc/blockchain.h \ @@ -206,6 +212,7 @@ BITCOIN_CORE_H = \ undo.h \ util/bip32.h \ util/bytevectorhash.h \ + util/check.h \ util/error.h \ util/fees.h \ util/spanparsing.h \ @@ -214,6 +221,7 @@ BITCOIN_CORE_H = \ util/memory.h \ util/moneystr.h \ util/rbf.h \ + util/settings.h \ util/string.h \ util/threadnames.h \ util/time.h \ @@ -235,6 +243,7 @@ BITCOIN_CORE_H = \ wallet/load.h \ wallet/psbtwallet.h \ wallet/rpcwallet.h \ + wallet/scriptpubkeyman.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/wallettool.h \ @@ -283,6 +292,7 @@ libbitcoin_server_a_SOURCES = \ net_processing.cpp \ node/coin.cpp \ node/coinstats.cpp \ + node/context.cpp \ node/psbt.cpp \ node/transaction.cpp \ noui.cpp \ @@ -338,11 +348,11 @@ libbitcoin_wallet_a_SOURCES = \ wallet/db.cpp \ wallet/feebumper.cpp \ wallet/fees.cpp \ - wallet/ismine.cpp \ wallet/load.cpp \ wallet/psbtwallet.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ + wallet/scriptpubkeyman.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ wallet/walletutil.cpp \ @@ -495,6 +505,7 @@ libbitcoin_util_a_SOURCES = \ interfaces/handler.cpp \ logging.cpp \ random.cpp \ + randomenv.cpp \ rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ @@ -506,6 +517,7 @@ libbitcoin_util_a_SOURCES = \ util/system.cpp \ util/moneystr.cpp \ util/rbf.cpp \ + util/settings.cpp \ util/threadnames.cpp \ util/spanparsing.cpp \ util/strencodings.cpp \ @@ -701,12 +713,6 @@ if HARDEN $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) endif -if ENABLE_BIP70 -%.pb.cc %.pb.h: %.proto - @test -f $(PROTOC) - $(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$(<D) $< -endif - if EMBEDDED_LEVELDB include Makefile.leveldb.include endif diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index e421b377a0..c9e4fcc4bc 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -30,6 +30,7 @@ bench_bench_bitcoin_SOURCES = \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/mempool_stress.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ @@ -39,8 +40,10 @@ bench_bench_bitcoin_SOURCES = \ bench/lockedpool.cpp \ bench/poly1305.cpp \ bench/prevector.cpp \ - test/setup_common.h \ - test/setup_common.cpp \ + test/util/transaction_utils.h \ + test/util/transaction_utils.cpp \ + test/util/setup_common.h \ + test/util/setup_common.cpp \ test/util.h \ test/util.cpp diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 9ab7f02e22..13b1470b58 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -95,16 +95,6 @@ QT_QRC = qt/bitcoin.qrc QT_QRC_LOCALE_CPP = qt/qrc_bitcoin_locale.cpp QT_QRC_LOCALE = qt/bitcoin_locale.qrc -if ENABLE_BIP70 -PROTOBUF_CC = qt/paymentrequest.pb.cc -PROTOBUF_H = qt/paymentrequest.pb.h -PROTOBUF_PROTO = qt/paymentrequest.proto -else -PROTOBUF_CC = -PROTOBUF_H = -PROTOBUF_PROTO = -endif - BITCOIN_QT_H = \ qt/addressbookpage.h \ qt/addresstablemodel.h \ @@ -134,7 +124,6 @@ BITCOIN_QT_H = \ qt/optionsdialog.h \ qt/optionsmodel.h \ qt/overviewpage.h \ - qt/paymentrequestplus.h \ qt/paymentserver.h \ qt/peertablemodel.h \ qt/platformstyle.h \ @@ -269,18 +258,12 @@ BITCOIN_QT_WALLET_CPP = \ qt/walletmodeltransaction.cpp \ qt/walletview.cpp -BITCOIN_QT_WALLET_BIP70_CPP = \ - qt/paymentrequestplus.cpp - BITCOIN_QT_CPP = $(BITCOIN_QT_BASE_CPP) if TARGET_WINDOWS BITCOIN_QT_CPP += $(BITCOIN_QT_WINDOWS_CPP) endif if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) -if ENABLE_BIP70 -BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_BIP70_CPP) -endif # ENABLE_BIP70 endif # ENABLE_WALLET RES_IMAGES = @@ -292,18 +275,17 @@ BITCOIN_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS qt_libbitcoinqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS) + $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(QR_CFLAGS) qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif -nodist_qt_libbitcoinqt_a_SOURCES = $(QT_MOC_CPP) $(QT_MOC) $(PROTOBUF_CC) \ - $(PROTOBUF_H) $(QT_QRC_CPP) $(QT_QRC_LOCALE_CPP) +nodist_qt_libbitcoinqt_a_SOURCES = $(QT_MOC_CPP) $(QT_MOC) $(QT_QRC_CPP) $(QT_QRC_LOCALE_CPP) # forms/foo.h -> forms/ui_foo.h QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI:.ui=.h)))) @@ -313,14 +295,9 @@ QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI: $(QT_MOC): $(QT_FORMS_H) $(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) : | $(QT_MOC) -#Generating these with a half-written protobuf header leads to wacky results. -#This makes sure it's done. -$(QT_MOC): $(PROTOBUF_H) -$(QT_MOC_CPP): $(PROTOBUF_H) - # bitcoin-qt binary # qt_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS) + $(QT_INCLUDES) $(QR_CFLAGS) qt_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_bitcoin_qt_SOURCES = qt/main.cpp @@ -335,15 +312,8 @@ if ENABLE_ZMQ qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ - $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -if ENABLE_BIP70 -qt_bitcoin_qt_LDADD += $(SSL_LIBS) -else -if TARGET_WINDOWS -qt_bitcoin_qt_LDADD += $(SSL_LIBS) -endif -endif qt_bitcoin_qt_LDADD += $(CRYPTO_LIBS) qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_bitcoin_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX @@ -368,7 +338,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ @rm $(@D)/temp_$(<F) -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) $(PROTOBUF_H) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 4acfff809e..562b393b22 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -15,10 +15,6 @@ if ENABLE_WALLET TEST_QT_MOC_CPP += \ qt/test/moc_addressbooktests.cpp \ qt/test/moc_wallettests.cpp -if ENABLE_BIP70 -TEST_QT_MOC_CPP += \ - qt/test/moc_paymentservertests.cpp -endif # ENABLE_BIP70 endif # ENABLE_WALLET TEST_QT_H = \ @@ -28,18 +24,16 @@ TEST_QT_H = \ qt/test/rpcnestedtests.h \ qt/test/uritests.h \ qt/test/util.h \ - qt/test/paymentrequestdata.h \ - qt/test/paymentservertests.h \ qt/test/wallettests.h TEST_BITCOIN_CPP = \ - test/setup_common.cpp + test/util/setup_common.cpp TEST_BITCOIN_H = \ - test/setup_common.h + test/util/setup_common.h qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(PROTOBUF_CFLAGS) + $(QT_INCLUDES) $(QT_TEST_INCLUDES) qt_test_test_bitcoin_qt_SOURCES = \ qt/test/apptests.cpp \ @@ -56,10 +50,6 @@ qt_test_test_bitcoin_qt_SOURCES += \ qt/test/addressbooktests.cpp \ qt/test/wallettests.cpp \ wallet/test/wallet_test_fixture.cpp -if ENABLE_BIP70 -qt_test_test_bitcoin_qt_SOURCES += \ - qt/test/paymentservertests.cpp -endif # ENABLE_BIP70 endif # ENABLE_WALLET nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) @@ -73,7 +63,7 @@ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ - $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(QR_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 7292ca0784..dd1ade5496 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -7,6 +7,7 @@ FUZZ_TARGETS = \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ test/fuzz/banentry_deserialize \ + test/fuzz/bech32 \ test/fuzz/block_deserialize \ test/fuzz/blockheader_deserialize \ test/fuzz/blocklocator_deserialize \ @@ -16,11 +17,14 @@ FUZZ_TARGETS = \ test/fuzz/blockundo_deserialize \ test/fuzz/bloomfilter_deserialize \ test/fuzz/coins_deserialize \ + test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ test/fuzz/inv_deserialize \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ + test/fuzz/parse_iso8601 \ + test/fuzz/script \ test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/spanparsing \ @@ -53,16 +57,26 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) BITCOIN_TEST_SUITE = \ + test/util/blockfilter.cpp \ + test/util/blockfilter.h \ + test/util/logging.cpp \ + test/util/logging.h \ + test/util/transaction_utils.cpp \ + test/util/transaction_utils.h \ test/main.cpp \ - test/setup_common.h \ - test/setup_common.cpp + test/util/setup_common.h \ + test/util/setup_common.cpp \ + test/util/str.h \ + test/util/str.cpp FUZZ_SUITE = \ - test/setup_common.h \ - test/setup_common.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h \ - test/fuzz/FuzzedDataProvider.h + test/fuzz/FuzzedDataProvider.h \ + test/util/setup_common.cpp \ + test/util/setup_common.h \ + test/util/str.cpp \ + test/util/str.h FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_SERVER) \ @@ -113,6 +127,7 @@ BITCOIN_TESTS =\ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ + test/logging_tests.cpp \ test/dbwrapper_tests.cpp \ test/validation_tests.cpp \ test/mempool_tests.cpp \ @@ -137,6 +152,7 @@ BITCOIN_TESTS =\ test/script_standard_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ + test/settings_tests.cpp \ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ @@ -235,6 +251,12 @@ test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bech32_SOURCES = $(FUZZ_SUITE) test/fuzz/bech32.cpp +test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) + test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -253,12 +275,30 @@ test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_descriptor_parse_SOURCES = $(FUZZ_SUITE) test/fuzz/descriptor_parse.cpp +test_fuzz_descriptor_parse_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_descriptor_parse_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_descriptor_parse_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_descriptor_parse_LDADD = $(FUZZ_SUITE_LD_COMMON) + test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_iso8601_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_iso8601.cpp +test_fuzz_parse_iso8601_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_iso8601_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_script_SOURCES = $(FUZZ_SUITE) test/fuzz/script.cpp +test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) + test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/addrdb.h b/src/addrdb.h index 290b63dd12..ad85224d1f 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -7,6 +7,7 @@ #define BITCOIN_ADDRDB_H #include <fs.h> +#include <net_types.h> // For banmap_t #include <serialize.h> #include <string> @@ -79,8 +80,6 @@ public: } }; -typedef std::map<CSubNet, CBanEntry> banmap_t; - /** Access to the (IP) address database (peers.dat) */ class CAddrDB { diff --git a/src/banman.h b/src/banman.h index a1a00309dd..7943f666e8 100644 --- a/src/banman.h +++ b/src/banman.h @@ -10,6 +10,7 @@ #include <addrdb.h> #include <fs.h> +#include <net_types.h> // For banmap_t #include <sync.h> // NOTE: When adjusting this, update rpcnet:setban's help ("24h") @@ -66,5 +67,4 @@ private: const int64_t m_default_ban_time; }; -extern std::unique_ptr<BanMan> g_banman; #endif diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index f2b520e893..cc159eb191 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,7 +5,7 @@ #include <bench/bench.h> #include <chainparams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <validation.h> #include <algorithm> @@ -112,7 +112,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double printer.header(); for (const auto& p : benchmarks()) { - TestingSetup test{CBaseChainParams::REGTEST}; + RegTestingSetup test{}; { LOCK(cs_main); assert(::ChainActive().Height() == 0); diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index d0d7c03ee1..9235d5fe6a 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -36,7 +36,7 @@ int main(int argc, char** argv) SetupBenchArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } @@ -51,9 +51,16 @@ int main(int argc, char** argv) std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); bool is_list_only = gArgs.GetBoolArg("-list", false); + if (evaluations == 0) { + return EXIT_SUCCESS; + } else if (evaluations < 0) { + tfm::format(std::cerr, "Error parsing evaluations argument: %d\n", evaluations); + return EXIT_FAILURE; + } + double scaling_factor; if (!ParseDouble(scaling_str, &scaling_factor)) { - tfm::format(std::cerr, "Error parsing scaling factor as double: %s\n", scaling_str.c_str()); + tfm::format(std::cerr, "Error parsing scaling factor as double: %s\n", scaling_str); return EXIT_FAILURE; } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 157f936a95..2f47398d99 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -38,8 +38,8 @@ static void AssembleBlock(benchmark::State& state) LOCK(::cs_main); // Required for ::AcceptToMemoryPool. for (const auto& txr : txs) { - CValidationState state; - bool ret{::AcceptToMemoryPool(::mempool, state, txr, nullptr /* pfMissingInputs */, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; + TxValidationState state; + bool ret{::AcceptToMemoryPool(::mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; assert(ret); } } diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 4b13381e16..edf43bd4dc 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -42,7 +42,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); - CValidationState validationState; + BlockValidationState validationState; bool checked = CheckBlock(block, validationState, chainParams->GetConsensus()); assert(checked); } diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f2ab03e20e..29a145bfe6 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/context.h> #include <wallet/coinselection.h> #include <wallet/wallet.h> @@ -28,7 +29,8 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<st // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CoinSelection(benchmark::State& state) { - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); const CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -60,7 +62,8 @@ static void CoinSelection(benchmark::State& state) } typedef std::set<CInputCoin> CoinSet; -static auto testChain = interfaces::MakeChain(); +static NodeContext testNode; +static auto testChain = interfaces::MakeChain(testNode); static const CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxn; diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 6cfa3750d6..a783370b4e 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -54,7 +54,7 @@ static void DuplicateInputs(benchmark::State& state) block.hashMerkleRoot = BlockMerkleRoot(block); while (state.KeepRunning()) { - CValidationState cvstate{}; + BlockValidationState cvstate{}; assert(!CheckBlock(block, cvstate, chainparams.GetConsensus(), false, false)); assert(cvstate.GetRejectReason() == "bad-txns-inputs-duplicate"); } diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp new file mode 100644 index 0000000000..389e2c096f --- /dev/null +++ b/src/bench/mempool_stress.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <policy/policy.h> +#include <txmempool.h> + +#include <vector> + +static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + int64_t nTime = 0; + unsigned int nHeight = 1; + bool spendsCoinbase = false; + unsigned int sigOpCost = 4; + LockPoints lp; + pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, spendsCoinbase, sigOpCost, lp)); +} + +struct Available { + CTransactionRef ref; + size_t vin_left{0}; + size_t tx_count; + Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){} +}; + +static void ComplexMemPool(benchmark::State& state) +{ + FastRandomContext det_rand{true}; + std::vector<Available> available_coins; + std::vector<CTransactionRef> ordered_coins; + // Create some base transactions + size_t tx_counter = 1; + for (auto x = 0; x < 100; ++x) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << CScriptNum(tx_counter); + tx.vin[0].scriptWitness.stack.push_back(CScriptNum(x).getvch()); + tx.vout.resize(det_rand.randrange(10)+2); + for (auto& out : tx.vout) { + out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL; + out.nValue = 10 * COIN; + } + ordered_coins.emplace_back(MakeTransactionRef(tx)); + available_coins.emplace_back(ordered_coins.back(), tx_counter++); + } + for (auto x = 0; x < 800 && !available_coins.empty(); ++x) { + CMutableTransaction tx = CMutableTransaction(); + size_t n_ancestors = det_rand.randrange(10)+1; + for (size_t ancestor = 0; ancestor < n_ancestors && !available_coins.empty(); ++ancestor){ + size_t idx = det_rand.randrange(available_coins.size()); + Available coin = available_coins[idx]; + uint256 hash = coin.ref->GetHash(); + // biased towards taking just one ancestor, but maybe more + size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left); + for (size_t i = 0; i < n_to_take; ++i) { + tx.vin.emplace_back(); + tx.vin.back().prevout = COutPoint(hash, coin.vin_left++); + tx.vin.back().scriptSig = CScript() << coin.tx_count; + tx.vin.back().scriptWitness.stack.push_back(CScriptNum(coin.tx_count).getvch()); + } + if (coin.vin_left == coin.ref->vin.size()) { + coin = available_coins.back(); + available_coins.pop_back(); + } + tx.vout.resize(det_rand.randrange(10)+2); + for (auto& out : tx.vout) { + out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL; + out.nValue = 10 * COIN; + } + } + ordered_coins.emplace_back(MakeTransactionRef(tx)); + available_coins.emplace_back(ordered_coins.back(), tx_counter++); + } + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + while (state.KeepRunning()) { + for (auto& tx : ordered_coins) { + AddTx(tx, pool); + } + pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); + pool.TrimToSize(GetVirtualTransactionSize(*ordered_coins.front())); + } +} + +BENCHMARK(ComplexMemPool, 1); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 4891c57b3a..1c025e29d3 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -10,44 +10,10 @@ #include <script/script.h> #include <script/standard.h> #include <streams.h> +#include <test/util/transaction_utils.h> #include <array> -// FIXME: Dedup with BuildCreditingTransaction in test/script_tests.cpp. -static CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey) -{ - CMutableTransaction txCredit; - txCredit.nVersion = 1; - txCredit.nLockTime = 0; - txCredit.vin.resize(1); - txCredit.vout.resize(1); - txCredit.vin[0].prevout.SetNull(); - txCredit.vin[0].scriptSig = CScript() << CScriptNum(0) << CScriptNum(0); - txCredit.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txCredit.vout[0].scriptPubKey = scriptPubKey; - txCredit.vout[0].nValue = 1; - - return txCredit; -} - -// FIXME: Dedup with BuildSpendingTransaction in test/script_tests.cpp. -static CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CMutableTransaction& txCredit) -{ - CMutableTransaction txSpend; - txSpend.nVersion = 1; - txSpend.nLockTime = 0; - txSpend.vin.resize(1); - txSpend.vout.resize(1); - txSpend.vin[0].prevout.hash = txCredit.GetHash(); - txSpend.vin[0].prevout.n = 0; - txSpend.vin[0].scriptSig = scriptSig; - txSpend.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txSpend.vout[0].scriptPubKey = CScript(); - txSpend.vout[0].nValue = txCredit.vout[0].nValue; - - return txSpend; -} - // Microbenchmark for verification of a basic P2WPKH script. Can be easily // modified to measure performance of other types of scripts. static void VerifyScriptBench(benchmark::State& state) @@ -71,8 +37,8 @@ static void VerifyScriptBench(benchmark::State& state) CScript scriptPubKey = CScript() << witnessversion << ToByteVector(pubkeyHash); CScript scriptSig; CScript witScriptPubkey = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkeyHash) << OP_EQUALVERIFY << OP_CHECKSIG; - const CMutableTransaction& txCredit = BuildCreditingTransaction(scriptPubKey); - CMutableTransaction txSpend = BuildSpendingTransaction(scriptSig, txCredit); + const CMutableTransaction& txCredit = BuildCreditingTransaction(scriptPubKey, 1); + CMutableTransaction txSpend = BuildSpendingTransaction(scriptSig, CScriptWitness(), CTransaction(txCredit)); CScriptWitness& witness = txSpend.vin[0].scriptWitness; witness.stack.emplace_back(); key.Sign(SignatureHash(witScriptPubkey, txSpend, 0, SIGHASH_ALL, txCredit.vout[0].nValue, SigVersion::WITNESS_V0), witness.stack.back()); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 313b5a3ba0..0e660d6bcd 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/context.h> #include <optional.h> #include <test/util.h> #include <validationinterface.h> @@ -13,7 +14,8 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b { const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(); + NodeContext node; + std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); CWallet wallet{chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; { bool first_run; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 93b7a7152c..592fcbe8dd 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -105,7 +105,7 @@ static int AppInitRPC(int argc, char* argv[]) SetupCliArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { @@ -119,7 +119,7 @@ static int AppInitRPC(int argc, char* argv[]) strUsage += "\n" + gArgs.GetHelpMessage(); } - tfm::format(std::cout, "%s", strUsage.c_str()); + tfm::format(std::cout, "%s", strUsage); if (argc < 2) { tfm::format(std::cerr, "Error: too few parameters\n"); return EXIT_FAILURE; @@ -127,11 +127,11 @@ static int AppInitRPC(int argc, char* argv[]) return EXIT_SUCCESS; } if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "").c_str()); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); return EXIT_FAILURE; } if (!gArgs.ReadConfigFiles(error, true)) { - tfm::format(std::cerr, "Error reading configuration file: %s\n", error.c_str()); + tfm::format(std::cerr, "Error reading configuration file: %s\n", error); return EXIT_FAILURE; } // Check for -chain, -testnet or -regtest parameter (BaseParams() calls are only valid after this clause) @@ -258,12 +258,14 @@ public: result.pushKV("version", batch[ID_NETWORKINFO]["result"]["version"]); result.pushKV("protocolversion", batch[ID_NETWORKINFO]["result"]["protocolversion"]); result.pushKV("blocks", batch[ID_BLOCKCHAININFO]["result"]["blocks"]); + result.pushKV("headers", batch[ID_BLOCKCHAININFO]["result"]["headers"]); + result.pushKV("verificationprogress", batch[ID_BLOCKCHAININFO]["result"]["verificationprogress"]); result.pushKV("timeoffset", batch[ID_NETWORKINFO]["result"]["timeoffset"]); result.pushKV("connections", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); - if (!batch[ID_WALLETINFO].isNull()) { + if (!batch[ID_WALLETINFO]["result"].isNull()) { result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); result.pushKV("keypoololdest", batch[ID_WALLETINFO]["result"]["keypoololdest"]); @@ -366,7 +368,7 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co std::string endpoint = "/"; if (!gArgs.GetArgs("-rpcwallet").empty()) { std::string walletName = gArgs.GetArg("-rpcwallet", ""); - char *encodedURI = evhttp_uriencode(walletName.c_str(), walletName.size(), false); + char *encodedURI = evhttp_uriencode(walletName.data(), walletName.size(), false); if (encodedURI) { endpoint = "/wallet/"+ std::string(encodedURI); free(encodedURI); @@ -393,7 +395,7 @@ static UniValue CallRPC(BaseRequestHandler *rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string().c_str())); + GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string())); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } @@ -541,7 +543,7 @@ static int CommandLineRPC(int argc, char *argv[]) } if (strPrint != "") { - tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint.c_str()); + tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint); } return nRet; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index cabea610f3..c7af7e0fc8 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -83,7 +83,7 @@ static int AppInitRawTx(int argc, char* argv[]) SetupBitcoinTxArgs(); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } @@ -105,7 +105,7 @@ static int AppInitRawTx(int argc, char* argv[]) "\n"; strUsage += gArgs.GetHelpMessage(); - tfm::format(std::cout, "%s", strUsage.c_str()); + tfm::format(std::cout, "%s", strUsage); if (argc < 2) { tfm::format(std::cerr, "Error: too few parameters\n"); @@ -724,21 +724,21 @@ static void OutputTxJSON(const CTransaction& tx) TxToUniv(tx, uint256(), entry); std::string jsonOutput = entry.write(4); - tfm::format(std::cout, "%s\n", jsonOutput.c_str()); + tfm::format(std::cout, "%s\n", jsonOutput); } static void OutputTxHash(const CTransaction& tx) { std::string strHexHash = tx.GetHash().GetHex(); // the hex-encoded transaction hash (aka the transaction id) - tfm::format(std::cout, "%s\n", strHexHash.c_str()); + tfm::format(std::cout, "%s\n", strHexHash); } static void OutputTxHex(const CTransaction& tx) { std::string strHex = EncodeHexTx(tx); - tfm::format(std::cout, "%s\n", strHex.c_str()); + tfm::format(std::cout, "%s\n", strHex); } static void OutputTx(const CTransaction& tx) @@ -829,7 +829,7 @@ static int CommandLineRawTx(int argc, char* argv[]) } if (strPrint != "") { - tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint.c_str()); + tfm::format(nRet == 0 ? std::cout : std::cerr, "%s\n", strPrint); } return nRet; } diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index eda4f8ce78..917ecd71c5 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -36,7 +36,7 @@ static bool WalletAppInit(int argc, char* argv[]) SetupWalletToolArgs(); std::string error_message; if (!gArgs.ParseParameters(argc, argv, error_message)) { - tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message.c_str()); + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } if (argc < 2 || HelpRequested(gArgs)) { @@ -48,7 +48,7 @@ static bool WalletAppInit(int argc, char* argv[]) " bitcoin-wallet [options] <command>\n\n" + gArgs.GetHelpMessage(); - tfm::format(std::cout, "%s", usage.c_str()); + tfm::format(std::cout, "%s", usage); return false; } @@ -56,7 +56,7 @@ static bool WalletAppInit(int argc, char* argv[]) LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", gArgs.GetBoolArg("-debug", false)); if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "").c_str()); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); return false; } // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause) @@ -87,7 +87,7 @@ int main(int argc, char* argv[]) for(int i = 1; i < argc; ++i) { if (!IsSwitchChar(argv[i][0])) { if (!method.empty()) { - tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method.c_str(), argv[i]); + tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method, argv[i]); return EXIT_FAILURE; } method = argv[i]; diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index ddd6f8839c..4b5cea4849 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -12,6 +12,7 @@ #include <compat.h> #include <init.h> #include <interfaces/chain.h> +#include <node/context.h> #include <noui.h> #include <shutdown.h> #include <ui_interface.h> @@ -24,13 +25,13 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -static void WaitForShutdown() +static void WaitForShutdown(NodeContext& node) { while (!ShutdownRequested()) { MilliSleep(200); } - Interrupt(); + Interrupt(node); } ////////////////////////////////////////////////////////////////////////////// @@ -39,8 +40,8 @@ static void WaitForShutdown() // static bool AppInit(int argc, char* argv[]) { - InitInterfaces interfaces; - interfaces.chain = interfaces::MakeChain(); + NodeContext node; + node.chain = interfaces::MakeChain(node); bool fRet = false; @@ -70,7 +71,7 @@ static bool AppInit(int argc, char* argv[]) strUsage += "\n" + gArgs.GetHelpMessage(); } - tfm::format(std::cout, "%s", strUsage.c_str()); + tfm::format(std::cout, "%s", strUsage); return true; } @@ -142,7 +143,7 @@ static bool AppInit(int argc, char* argv[]) // If locking the data directory failed, exit immediately return false; } - fRet = AppInitMain(interfaces); + fRet = AppInitMain(node); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); @@ -152,11 +153,11 @@ static bool AppInit(int argc, char* argv[]) if (!fRet) { - Interrupt(); + Interrupt(node); } else { - WaitForShutdown(); + WaitForShutdown(node); } - Shutdown(interfaces); + Shutdown(node); return fRet; } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index f0fcf675eb..bf13297582 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -197,13 +197,13 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector< if (vtx_missing.size() != tx_missing_offset) return READ_STATUS_INVALID; - CValidationState state; + BlockValidationState state; if (!CheckBlock(block, state, Params().GetConsensus())) { // TODO: We really want to just check merkle tree manually here, // but that is expensive, and CheckBlock caches a block's // "checked-status" (in the CBlock?). CBlock should be able to // check its own merkle root and cache that check. - if (state.GetReason() == ValidationInvalidReason::BLOCK_MUTATED) + if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) return READ_STATUS_FAILED; // Possible Short ID collision return READ_STATUS_CHECKBLOCK_FAILED; } diff --git a/src/bloom.h b/src/bloom.h index 7d3aa878b0..c3f64ba4bc 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -115,9 +115,6 @@ public: class CRollingBloomFilter { public: - // A random bloom filter calls GetRand() at creation time. - // Don't create global CRollingBloomFilter objects, as they may be - // constructed before the randomizer is properly initialized. CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 7ff30ac1cd..0a1cb858ef 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -62,7 +62,7 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits class CMainParams : public CChainParams { public: CMainParams() { - strNetworkID = "main"; + strNetworkID = CBaseChainParams::MAIN; consensus.nSubsidyHalvingInterval = 210000; consensus.BIP16Exception = uint256S("0x00000000000002dc756eebf4f49723ed8d30cc28a5f108eb94b1ba88ac4f9c22"); consensus.BIP34Height = 227931; @@ -71,7 +71,7 @@ public: consensus.BIP66Height = 363725; // 00000000000000000379eaa19dce8c9b722d46ae6a57c2f1a988119488b50931 consensus.CSVHeight = 419328; // 000000000000000004a1b34462cb8aeebd5799177f7a29cf28f2d1961716b5b5 consensus.SegwitHeight = 481824; // 0000000000000000001c8018d9cb3b742ef25114f27563e3fc4a1902167f9893 - consensus.MinBIP9WarningHeight = consensus.SegwitHeight + consensus.nMinerConfirmationWindow; + consensus.MinBIP9WarningHeight = 483840; // segwit activation height + miner confirmation window consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -169,7 +169,7 @@ public: class CTestNetParams : public CChainParams { public: CTestNetParams() { - strNetworkID = "test"; + strNetworkID = CBaseChainParams::TESTNET; consensus.nSubsidyHalvingInterval = 210000; consensus.BIP16Exception = uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105"); consensus.BIP34Height = 21111; @@ -178,7 +178,7 @@ public: consensus.BIP66Height = 330776; // 000000002104c8c45e99a8853285a3b592602a3ccde2b832481da85e9e4ba182 consensus.CSVHeight = 770112; // 00000000025e930139bac5c6c31a403776da130831ab85be56578f3fa75369bb consensus.SegwitHeight = 834624; // 00000000002b980fcd729daaa248fd9316a5200e9b367f4ff2c42453e84201ca - consensus.MinBIP9WarningHeight = consensus.SegwitHeight + consensus.nMinerConfirmationWindow; + consensus.MinBIP9WarningHeight = 836640; // segwit activation height + miner confirmation window consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -254,7 +254,7 @@ public: class CRegTestParams : public CChainParams { public: explicit CRegTestParams(const ArgsManager& args) { - strNetworkID = "regtest"; + strNetworkID = CBaseChainParams::REGTEST; consensus.nSubsidyHalvingInterval = 150; consensus.BIP16Exception = uint256(); consensus.BIP34Height = 500; // BIP34 activated on regtest (Used in functional tests) diff --git a/src/chainparams.h b/src/chainparams.h index 8f1d27e03c..6be066806b 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -75,7 +75,7 @@ public: uint64_t AssumedChainStateSize() const { return m_assumed_chain_state_size; } /** Whether it is possible to mine blocks on demand (no retargeting) */ bool MineBlocksOnDemand() const { return consensus.fPowNoRetargeting; } - /** Return the BIP70 network string (main, test or regtest) */ + /** Return the network string */ std::string NetworkIDString() const { return strNetworkID; } /** Return the list of hostnames to look up for DNS seeds */ const std::vector<std::string>& DNSSeeds() const { return vSeeds; } @@ -120,7 +120,7 @@ std::unique_ptr<const CChainParams> CreateChainParams(const std::string& chain); const CChainParams &Params(); /** - * Sets the params returned by Params() to those for the given BIP70 chain name. + * Sets the params returned by Params() to those for the given chain name. * @throws std::runtime_error when the chain is not supported. */ void SelectParams(const std::string& chain); diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index f34646f7ac..69fe2438f3 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -15,10 +15,12 @@ class CBaseChainParams { public: - /** BIP70 chain name strings (main, test or regtest) */ + ///@{ + /** Chain name strings */ static const std::string MAIN; static const std::string TESTNET; static const std::string REGTEST; + ///@} const std::string& DataDir() const { return strDataDir; } int RPCPort() const { return nRPCPort; } diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index fe47f4899f..c254fe7cbf 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -17,20 +17,13 @@ #if defined(MAC_OSX) -#if !defined(bswap_16) - -// Mac OS X / Darwin features; we include a check for bswap_16 because if it is already defined, protobuf has -// defined these macros for us already; if it isn't, we do it ourselves. In either case, we get the exact same -// result regardless which path was taken #include <libkern/OSByteOrder.h> #define bswap_16(x) OSSwapInt16(x) #define bswap_32(x) OSSwapInt32(x) #define bswap_64(x) OSSwapInt64(x) -#endif // !defined(bswap_16) - #else -// Non-Mac OS X / non-Darwin +// Non-MacOS / non-Darwin #if HAVE_DECL_BSWAP_16 == 0 inline uint16_t bswap_16(uint16_t x) diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h new file mode 100644 index 0000000000..0877ad47d3 --- /dev/null +++ b/src/compat/cpuid.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_CPUID_H +#define BITCOIN_COMPAT_CPUID_H + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#define HAVE_GETCPUID + +#include <cpuid.h> + +// We can't use cpuid.h's __get_cpuid as it does not support subleafs. +void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + +#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#endif // BITCOIN_COMPAT_CPUID_H diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index 00ebbbd1ab..88bb12c713 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -7,50 +7,51 @@ #include <primitives/transaction.h> #include <consensus/validation.h> -bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) +bool CheckTransaction(const CTransaction& tx, TxValidationState& state) { // Basic checks that don't depend on any context if (tx.vin.empty()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vin-empty"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vin-empty"); if (tx.vout.empty()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vout-empty"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-empty"); // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-oversize"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-oversize"); // Check for negative or overflow output values (see CVE-2010-5139) CAmount nValueOut = 0; for (const auto& txout : tx.vout) { if (txout.nValue < 0) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vout-negative"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-negative"); if (txout.nValue > MAX_MONEY) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-vout-toolarge"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-toolarge"); nValueOut += txout.nValue; if (!MoneyRange(nValueOut)) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-txouttotal-toolarge"); } - // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock - if (fCheckDuplicateInputs) { - std::set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (!vInOutPoints.insert(txin.prevout).second) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); - } + // Check for duplicate inputs (see CVE-2018-17144) + // While Consensus::CheckTxInputs does check if all inputs of a tx are available, and UpdateCoins marks all inputs + // of a tx as spent, it does not check if the tx has duplicate inputs. + // Failure to run this check will result in either a crash or an inflation bug, depending on the implementation of + // the underlying coins database. + std::set<COutPoint> vInOutPoints; + for (const auto& txin : tx.vin) { + if (!vInOutPoints.insert(txin.prevout).second) + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-inputs-duplicate"); } if (tx.IsCoinBase()) { if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-length"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-cb-length"); } else { for (const auto& txin : tx.vin) if (txin.prevout.IsNull()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-prevout-null"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-prevout-null"); } return true; diff --git a/src/consensus/tx_check.h b/src/consensus/tx_check.h index bcfdf36bf9..b818a284f1 100644 --- a/src/consensus/tx_check.h +++ b/src/consensus/tx_check.h @@ -13,8 +13,8 @@ */ class CTransaction; -class CValidationState; +class TxValidationState; -bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); +bool CheckTransaction(const CTransaction& tx, TxValidationState& state); #endif // BITCOIN_CONSENSUS_TX_CHECK_H diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 4b93cae848..31bdabea28 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -156,11 +156,11 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i return nSigOps; } -bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee) +bool Consensus::CheckTxInputs(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee) { // are the actual inputs available? if (!inputs.HaveInputs(tx)) { - return state.Invalid(ValidationInvalidReason::TX_MISSING_INPUTS, false, REJECT_INVALID, "bad-txns-inputs-missingorspent", + return state.Invalid(TxValidationResult::TX_MISSING_INPUTS, "bad-txns-inputs-missingorspent", strprintf("%s: inputs missing/spent", __func__)); } @@ -172,27 +172,27 @@ bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, c // If prev is coinbase, check that it's matured if (coin.IsCoinBase() && nSpendHeight - coin.nHeight < COINBASE_MATURITY) { - return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "bad-txns-premature-spend-of-coinbase", strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); } // Check for negative or overflow input values nValueIn += coin.out.nValue; if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-inputvalues-outofrange"); } } const CAmount value_out = tx.GetValueOut(); if (nValueIn < value_out) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-in-belowout", + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-in-belowout", strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(value_out))); } // Tally transaction fees const CAmount txfee_aux = nValueIn - value_out; if (!MoneyRange(txfee_aux)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-fee-outofrange"); } txfee = txfee_aux; diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index 3519fc555d..b6599f2878 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -13,7 +13,7 @@ class CBlockIndex; class CCoinsViewCache; class CTransaction; -class CValidationState; +class TxValidationState; /** Transaction validation functions */ @@ -24,7 +24,7 @@ namespace Consensus { * @param[out] txfee Set to the transaction fee if successful. * Preconditions: tx.IsCoinBase() is false. */ -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee); +bool CheckTxInputs(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee); } // namespace Consensus /** Auxiliary functions for transaction validation (ideally should not be exposed) */ diff --git a/src/consensus/validation.h b/src/consensus/validation.h index 2e23f4b3a4..e602b9d5f3 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -12,25 +12,12 @@ #include <primitives/transaction.h> #include <primitives/block.h> -/** "reject" message codes */ -static const unsigned char REJECT_MALFORMED = 0x01; -static const unsigned char REJECT_INVALID = 0x10; -static const unsigned char REJECT_OBSOLETE = 0x11; -static const unsigned char REJECT_DUPLICATE = 0x12; -static const unsigned char REJECT_NONSTANDARD = 0x40; -// static const unsigned char REJECT_DUST = 0x41; // part of BIP 61 -static const unsigned char REJECT_INSUFFICIENTFEE = 0x42; -static const unsigned char REJECT_CHECKPOINT = 0x43; - -/** A "reason" why something was invalid, suitable for determining whether the - * provider of the object should be banned/ignored/disconnected/etc. - * These are much more granular than the rejection codes, which may be more - * useful for some other use-cases. +/** A "reason" why a transaction was invalid, suitable for determining whether the + * provider of the transaction should be banned/ignored/disconnected/etc. */ -enum class ValidationInvalidReason { - // txn and blocks: - NONE, //!< not actually invalid - CONSENSUS, //!< invalid by consensus rules (excluding any below reasons) +enum class TxValidationResult { + TX_RESULT_UNSET, //!< initial value. Tx has not yet been rejected + TX_CONSENSUS, //!< invalid by consensus rules /** * Invalid by a change to consensus rules more recent than SegWit. * Currently unused as there are no such consensus rule changes, and any download @@ -38,18 +25,9 @@ enum class ValidationInvalidReason { * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork * is uninteresting. */ - RECENT_CONSENSUS_CHANGE, - // Only blocks (or headers): - CACHED_INVALID, //!< this object was cached as being invalid, but we don't know why - BLOCK_INVALID_HEADER, //!< invalid proof of work or time too old - BLOCK_MUTATED, //!< the block's data didn't match the data committed to by the PoW - BLOCK_MISSING_PREV, //!< We don't have the previous block the checked one is built on - BLOCK_INVALID_PREV, //!< A block this one builds on is invalid - BLOCK_TIME_FUTURE, //!< block timestamp was > 2 hours in the future (or our clock is bad) - BLOCK_CHECKPOINT, //!< the block failed to meet one of our checkpoints - // Only loose txn: + TX_RECENT_CONSENSUS_CHANGE, TX_NOT_STANDARD, //!< didn't meet our local policy rules - TX_MISSING_INPUTS, //!< a transaction was missing some of its inputs + TX_MISSING_INPUTS, //!< transaction was missing some of its inputs TX_PREMATURE_SPEND, //!< transaction spends a coinbase too early, or violates locktime/sequence locks /** * Transaction might be missing a witness, have a witness prior to SegWit @@ -60,85 +38,107 @@ enum class ValidationInvalidReason { /** * Tx already in mempool or conflicts with a tx in the chain * (if it conflicts with another tx in mempool, we use MEMPOOL_POLICY as it failed to reach the RBF threshold) - * TODO: Currently this is only used if the transaction already exists in the mempool or on chain, - * TODO: ATMP's fMissingInputs and a valid CValidationState being used to indicate missing inputs + * Currently this is only used if the transaction already exists in the mempool or on chain. */ TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits }; -inline bool IsTransactionReason(ValidationInvalidReason r) -{ - return r == ValidationInvalidReason::NONE || - r == ValidationInvalidReason::CONSENSUS || - r == ValidationInvalidReason::RECENT_CONSENSUS_CHANGE || - r == ValidationInvalidReason::TX_NOT_STANDARD || - r == ValidationInvalidReason::TX_PREMATURE_SPEND || - r == ValidationInvalidReason::TX_MISSING_INPUTS || - r == ValidationInvalidReason::TX_WITNESS_MUTATED || - r == ValidationInvalidReason::TX_CONFLICT || - r == ValidationInvalidReason::TX_MEMPOOL_POLICY; -} +/** A "reason" why a block was invalid, suitable for determining whether the + * provider of the block should be banned/ignored/disconnected/etc. + * These are much more granular than the rejection codes, which may be more + * useful for some other use-cases. + */ +enum class BlockValidationResult { + BLOCK_RESULT_UNSET, //!< initial value. Block has not yet been rejected + BLOCK_CONSENSUS, //!< invalid by consensus rules (excluding any below reasons) + /** + * Invalid by a change to consensus rules more recent than SegWit. + * Currently unused as there are no such consensus rule changes, and any download + * sources realistically need to support SegWit in order to provide useful data, + * so differentiating between always-invalid and invalid-by-pre-SegWit-soft-fork + * is uninteresting. + */ + BLOCK_RECENT_CONSENSUS_CHANGE, + BLOCK_CACHED_INVALID, //!< this block was cached as being invalid and we didn't store the reason why + BLOCK_INVALID_HEADER, //!< invalid proof of work or time too old + BLOCK_MUTATED, //!< the block's data didn't match the data committed to by the PoW + BLOCK_MISSING_PREV, //!< We don't have the previous block the checked one is built on + BLOCK_INVALID_PREV, //!< A block this one builds on is invalid + BLOCK_TIME_FUTURE, //!< block timestamp was > 2 hours in the future (or our clock is bad) + BLOCK_CHECKPOINT, //!< the block failed to meet one of our checkpoints +}; -inline bool IsBlockReason(ValidationInvalidReason r) -{ - return r == ValidationInvalidReason::NONE || - r == ValidationInvalidReason::CONSENSUS || - r == ValidationInvalidReason::RECENT_CONSENSUS_CHANGE || - r == ValidationInvalidReason::CACHED_INVALID || - r == ValidationInvalidReason::BLOCK_INVALID_HEADER || - r == ValidationInvalidReason::BLOCK_MUTATED || - r == ValidationInvalidReason::BLOCK_MISSING_PREV || - r == ValidationInvalidReason::BLOCK_INVALID_PREV || - r == ValidationInvalidReason::BLOCK_TIME_FUTURE || - r == ValidationInvalidReason::BLOCK_CHECKPOINT; -} -/** Capture information about block/transaction validation */ -class CValidationState { + +/** Base class for capturing information about block/transaction validation. This is subclassed + * by TxValidationState and BlockValidationState for validation information on transactions + * and blocks respectively. */ +class ValidationState { private: enum mode_state { MODE_VALID, //!< everything ok MODE_INVALID, //!< network rule violation (DoS value may be set) MODE_ERROR, //!< run-time error - } mode; - ValidationInvalidReason m_reason; - std::string strRejectReason; - unsigned int chRejectCode; - std::string strDebugMessage; -public: - CValidationState() : mode(MODE_VALID), m_reason(ValidationInvalidReason::NONE), chRejectCode(0) {} - bool Invalid(ValidationInvalidReason reasonIn, bool ret = false, - unsigned int chRejectCodeIn=0, const std::string &strRejectReasonIn="", - const std::string &strDebugMessageIn="") { - m_reason = reasonIn; - chRejectCode = chRejectCodeIn; - strRejectReason = strRejectReasonIn; - strDebugMessage = strDebugMessageIn; - if (mode == MODE_ERROR) - return ret; - mode = MODE_INVALID; - return ret; + } m_mode; + std::string m_reject_reason; + std::string m_debug_message; +protected: + void Invalid(const std::string &reject_reason="", + const std::string &debug_message="") + { + m_reject_reason = reject_reason; + m_debug_message = debug_message; + if (m_mode != MODE_ERROR) m_mode = MODE_INVALID; } - bool Error(const std::string& strRejectReasonIn) { - if (mode == MODE_VALID) - strRejectReason = strRejectReasonIn; - mode = MODE_ERROR; +public: + // ValidationState is abstract. Have a pure virtual destructor. + virtual ~ValidationState() = 0; + + ValidationState() : m_mode(MODE_VALID) {} + bool Error(const std::string& reject_reason) + { + if (m_mode == MODE_VALID) + m_reject_reason = reject_reason; + m_mode = MODE_ERROR; return false; } - bool IsValid() const { - return mode == MODE_VALID; - } - bool IsInvalid() const { - return mode == MODE_INVALID; + bool IsValid() const { return m_mode == MODE_VALID; } + bool IsInvalid() const { return m_mode == MODE_INVALID; } + bool IsError() const { return m_mode == MODE_ERROR; } + std::string GetRejectReason() const { return m_reject_reason; } + std::string GetDebugMessage() const { return m_debug_message; } +}; + +inline ValidationState::~ValidationState() {}; + +class TxValidationState : public ValidationState { +private: + TxValidationResult m_result; +public: + bool Invalid(TxValidationResult result, + const std::string &reject_reason="", + const std::string &debug_message="") + { + m_result = result; + ValidationState::Invalid(reject_reason, debug_message); + return false; } - bool IsError() const { - return mode == MODE_ERROR; + TxValidationResult GetResult() const { return m_result; } +}; + +class BlockValidationState : public ValidationState { +private: + BlockValidationResult m_result; +public: + bool Invalid(BlockValidationResult result, + const std::string &reject_reason="", + const std::string &debug_message="") { + m_result = result; + ValidationState::Invalid(reject_reason, debug_message); + return false; } - ValidationInvalidReason GetReason() const { return m_reason; } - unsigned int GetRejectCode() const { return chRejectCode; } - std::string GetRejectReason() const { return strRejectReason; } - std::string GetDebugMessage() const { return strDebugMessage; } + BlockValidationResult GetResult() const { return m_result; } }; // These implement the weight = (stripped_size * 4) + witness_size formula, diff --git a/src/crypto/hkdf_sha256_32.cpp b/src/crypto/hkdf_sha256_32.cpp index 9cea5995ec..e684eced37 100644 --- a/src/crypto/hkdf_sha256_32.cpp +++ b/src/crypto/hkdf_sha256_32.cpp @@ -9,7 +9,7 @@ CHKDF_HMAC_SHA256_L32::CHKDF_HMAC_SHA256_L32(const unsigned char* ikm, size_t ikmlen, const std::string& salt) { - CHMAC_SHA256((const unsigned char*)salt.c_str(), salt.size()).Write(ikm, ikmlen).Finalize(m_prk); + CHMAC_SHA256((const unsigned char*)salt.data(), salt.size()).Write(ikm, ikmlen).Finalize(m_prk); } void CHKDF_HMAC_SHA256_L32::Expand32(const std::string& info, unsigned char hash[OUTPUT_SIZE]) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 3257ee7f97..dda7e5230f 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -8,9 +8,10 @@ #include <assert.h> #include <string.h> +#include <compat/cpuid.h> + #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #if defined(USE_ASM) -#include <cpuid.h> namespace sha256_sse4 { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); @@ -546,18 +547,7 @@ bool SelfTest() { return true; } - #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) -// We can't use cpuid.h's __get_cpuid as it does not support subleafs. -void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} - /** Check whether the OS has enabled AVX registers. */ bool AVXEnabled() { @@ -572,7 +562,7 @@ bool AVXEnabled() std::string SHA256AutoDetect() { std::string ret = "standard"; -#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) +#if defined(USE_ASM) && defined(HAVE_GETCPUID) bool have_sse4 = false; bool have_xsave = false; bool have_avx = false; @@ -589,7 +579,7 @@ std::string SHA256AutoDetect() (void)enabled_avx; uint32_t eax, ebx, ecx, edx; - cpuid(1, 0, eax, ebx, ecx, edx); + GetCPUID(1, 0, eax, ebx, ecx, edx); have_sse4 = (ecx >> 19) & 1; have_xsave = (ecx >> 27) & 1; have_avx = (ecx >> 28) & 1; @@ -597,7 +587,7 @@ std::string SHA256AutoDetect() enabled_avx = AVXEnabled(); } if (have_sse4) { - cpuid(7, 0, eax, ebx, ecx, edx); + GetCPUID(7, 0, eax, ebx, ecx, edx); have_avx2 = (ebx >> 5) & 1; have_shani = (ebx >> 29) & 1; } diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index 4118ac1b18..fc7dd1b87e 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -23,6 +23,7 @@ public: CSHA512& Write(const unsigned char* data, size_t len); void Finalize(unsigned char hash[OUTPUT_SIZE]); CSHA512& Reset(); + uint64_t Size() const { return bytes; } }; #endif // BITCOIN_CRYPTO_SHA512_H diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 4d0b094fa2..674f47b956 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -14,42 +14,40 @@ #include <vector> -/** namespace CuckooCache provides high performance cache primitives +/** High-performance cache primitives. * * Summary: * - * 1) bit_packed_atomic_flags is bit-packed atomic flags for garbage collection + * 1. @ref bit_packed_atomic_flags is bit-packed atomic flags for garbage collection * - * 2) cache is a cache which is performant in memory usage and lookup speed. It - * is lockfree for erase operations. Elements are lazily erased on the next - * insert. + * 2. @ref cache is a cache which is performant in memory usage and lookup speed. It + * is lockfree for erase operations. Elements are lazily erased on the next insert. */ namespace CuckooCache { -/** bit_packed_atomic_flags implements a container for garbage collection flags +/** @ref bit_packed_atomic_flags implements a container for garbage collection flags * that is only thread unsafe on calls to setup. This class bit-packs collection * flags for memory efficiency. * - * All operations are std::memory_order_relaxed so external mechanisms must + * All operations are `std::memory_order_relaxed` so external mechanisms must * ensure that writes and reads are properly synchronized. * - * On setup(n), all bits up to n are marked as collected. + * On setup(n), all bits up to `n` are marked as collected. * * Under the hood, because it is an 8-bit type, it makes sense to use a multiple * of 8 for setup, but it will be safe if that is not the case as well. - * */ class bit_packed_atomic_flags { std::unique_ptr<std::atomic<uint8_t>[]> mem; public: - /** No default constructor as there must be some size */ + /** No default constructor, as there must be some size. */ bit_packed_atomic_flags() = delete; /** * bit_packed_atomic_flags constructor creates memory to sufficiently - * keep track of garbage collection information for size entries. + * keep track of garbage collection information for `size` entries. * * @param size the number of elements to allocate space for * @@ -68,7 +66,7 @@ public: }; /** setup marks all entries and ensures that bit_packed_atomic_flags can store - * at least size entries + * at least `b` entries. * * @param b the number of elements to allocate space for * @post bit_set, bit_unset, and bit_is_set function properly forall x. x < @@ -84,19 +82,18 @@ public: /** bit_set sets an entry as discardable. * - * @param s the index of the entry to bit_set. + * @param s the index of the entry to bit_set * @post immediately subsequent call (assuming proper external memory * ordering) to bit_is_set(s) == true. - * */ inline void bit_set(uint32_t s) { mem[s >> 3].fetch_or(1 << (s & 7), std::memory_order_relaxed); } - /** bit_unset marks an entry as something that should not be overwritten + /** bit_unset marks an entry as something that should not be overwritten. * - * @param s the index of the entry to bit_unset. + * @param s the index of the entry to bit_unset * @post immediately subsequent call (assuming proper external memory * ordering) to bit_is_set(s) == false. */ @@ -105,10 +102,10 @@ public: mem[s >> 3].fetch_and(~(1 << (s & 7)), std::memory_order_relaxed); } - /** bit_is_set queries the table for discardability at s + /** bit_is_set queries the table for discardability at `s`. * - * @param s the index of the entry to read. - * @returns if the bit at index s was set. + * @param s the index of the entry to read + * @returns true if the bit at index `s` was set, false otherwise * */ inline bool bit_is_set(uint32_t s) const { @@ -116,15 +113,15 @@ public: } }; -/** cache implements a cache with properties similar to a cuckoo-set +/** @ref cache implements a cache with properties similar to a cuckoo-set. * - * The cache is able to hold up to (~(uint32_t)0) - 1 elements. + * The cache is able to hold up to `(~(uint32_t)0) - 1` elements. * * Read Operations: - * - contains(*, false) + * - contains() for `erase=false` * * Read+Erase Operations: - * - contains(*, true) + * - contains() for `erase=true` * * Erase Operations: * - allow_erase() @@ -141,10 +138,10 @@ public: * * User Must Guarantee: * - * 1) Write Requires synchronized access (e.g., a lock) - * 2) Read Requires no concurrent Write, synchronized with the last insert. - * 3) Erase requires no concurrent Write, synchronized with last insert. - * 4) An Erase caller must release all memory before allowing a new Writer. + * 1. Write requires synchronized access (e.g. a lock) + * 2. Read requires no concurrent Write, synchronized with last insert. + * 3. Erase requires no concurrent Write, synchronized with last insert. + * 4. An Erase caller must release all memory before allowing a new Writer. * * * Note on function names: @@ -177,7 +174,7 @@ private: mutable std::vector<bool> epoch_flags; /** epoch_heuristic_counter is used to determine when an epoch might be aged - * & an expensive scan should be done. epoch_heuristic_counter is + * & an expensive scan should be done. epoch_heuristic_counter is * decremented on insert and reset to the new number of inserts which would * cause the epoch to reach epoch_size when it reaches zero. */ @@ -194,24 +191,25 @@ private: uint32_t epoch_size; /** depth_limit determines how many elements insert should try to replace. - * Should be set to log2(n)*/ + * Should be set to log2(n). + */ uint8_t depth_limit; /** hash_function is a const instance of the hash function. It cannot be * static or initialized at call time as it may have internal state (such as * a nonce). - * */ + */ const Hash hash_function; /** compute_hashes is convenience for not having to write out this * expression everywhere we use the hash values of an Element. * * We need to map the 32-bit input hash onto a hash bucket in a range [0, size) in a - * manner which preserves as much of the hash's uniformity as possible. Ideally + * manner which preserves as much of the hash's uniformity as possible. Ideally * this would be done by bitmasking but the size is usually not a power of two. * * The naive approach would be to use a mod -- which isn't perfectly uniform but so - * long as the hash is much larger than size it is not that bad. Unfortunately, + * long as the hash is much larger than size it is not that bad. Unfortunately, * mod/division is fairly slow on ordinary microprocessors (e.g. 90-ish cycles on * haswell, ARM doesn't even have an instruction for it.); when the divisor is a * constant the compiler will do clever tricks to turn it into a multiply+add+shift, @@ -223,10 +221,10 @@ private: * somewhat complicated and the result is still slower than other options: * * Instead we treat the 32-bit random number as a Q32 fixed-point number in the range - * [0,1) and simply multiply it by the size. Then we just shift the result down by - * 32-bits to get our bucket number. The result has non-uniformity the same as a + * [0, 1) and simply multiply it by the size. Then we just shift the result down by + * 32-bits to get our bucket number. The result has non-uniformity the same as a * mod, but it is much faster to compute. More about this technique can be found at - * http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + * http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ . * * The resulting non-uniformity is also more equally distributed which would be * advantageous for something like linear probing, though it shouldn't matter @@ -237,8 +235,8 @@ private: * 32*32->64 multiply, which means the operation is reasonably fast even on a * typical 32-bit processor. * - * @param e the element whose hashes will be returned - * @returns std::array<uint32_t, 8> of deterministic hashes derived from e + * @param e The element whose hashes will be returned + * @returns Deterministic hashes derived from `e` uniformly mapped onto the range [0, size) */ inline std::array<uint32_t, 8> compute_hashes(const Element& e) const { @@ -252,14 +250,14 @@ private: (uint32_t)(((uint64_t)hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}}; } - /* end - * @returns a constexpr index that can never be inserted to */ + /** invalid returns a special index that can never be inserted to + * @returns the special constexpr index that can never be inserted to */ constexpr uint32_t invalid() const { return ~(uint32_t)0; } - /** allow_erase marks the element at index n as discardable. Threadsafe + /** allow_erase marks the element at index `n` as discardable. Threadsafe * without any concurrent insert. * @param n the index to allow erasure of */ @@ -268,7 +266,7 @@ private: collection_flags.bit_set(n); } - /** please_keep marks the element at index n as an entry that should be kept. + /** please_keep marks the element at index `n` as an entry that should be kept. * Threadsafe without any concurrent insert. * @param n the index to prioritize keeping */ @@ -336,7 +334,7 @@ public: * * @param new_size the desired number of elements to store * @returns the maximum number of elements storable - **/ + */ uint32_t setup(uint32_t new_size) { // depth_limit must be at least one otherwise errors can occur. @@ -360,7 +358,7 @@ public: * negligible compared to the size of the elements. * * @param bytes the approximate number of bytes to use for this data - * structure. + * structure * @returns the maximum number of elements storable (see setup() * documentation for more detail) */ @@ -376,10 +374,12 @@ public: * It drops the last tried element if it runs out of depth before * encountering an open slot. * - * Thus + * Thus: * + * ``` * insert(x); * return contains(x, false); + * ``` * * is not guaranteed to return true. * @@ -387,7 +387,6 @@ public: * @post one of the following: All previously inserted elements and e are * now in the table, one previously inserted element is evicted from the * table, the entry attempted to be inserted is evicted. - * */ inline void insert(Element e) { @@ -416,9 +415,9 @@ public: /** Swap with the element at the location that was * not the last one looked at. Example: * - * 1) On first iteration, last_loc == invalid(), find returns last, so + * 1. On first iteration, last_loc == invalid(), find returns last, so * last_loc defaults to locs[0]. - * 2) On further iterations, where last_loc == locs[k], last_loc will + * 2. On further iterations, where last_loc == locs[k], last_loc will * go to locs[k+1 % 8], i.e., next of the 8 indices wrapping around * to 0 if needed. * @@ -439,17 +438,19 @@ public: } } - /* contains iterates through the hash locations for a given element + /** contains iterates through the hash locations for a given element * and checks to see if it is present. * * contains does not check garbage collected state (in other words, * garbage is only collected when the space is needed), so: * + * ``` * insert(x); * if (contains(x, true)) * return contains(x, false); * else * return true; + * ``` * * executed on a single thread will always return true! * @@ -458,7 +459,7 @@ public: * contains returns a bool set true if the element was found. * * @param e the element to check - * @param erase + * @param erase whether to attempt setting the garbage collect flag * * @post if erase is true and the element is found, then the garbage collect * flag is set diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 0edcb0286d..38b5b0efc4 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -19,7 +19,7 @@ public: bool HasWalletSupport() const override {return false;} void AddWalletOptions() const override; bool ParameterInteraction() const override {return true;} - void Construct(InitInterfaces& interfaces) const override {LogPrintf("No wallet support compiled in!\n");} + void Construct(NodeContext& node) const override {LogPrintf("No wallet support compiled in!\n");} }; void DummyWalletInit::AddWalletOptions() const diff --git a/src/fs.cpp b/src/fs.cpp index 7b422b8d70..73fb3b606e 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -107,10 +107,10 @@ std::string get_filesystem_error_message(const fs::filesystem_error& e) #else // Convert from Multi Byte to utf-16 std::string mb_string(e.what()); - int size = MultiByteToWideChar(CP_ACP, 0, mb_string.c_str(), mb_string.size(), nullptr, 0); + int size = MultiByteToWideChar(CP_ACP, 0, mb_string.data(), mb_string.size(), nullptr, 0); std::wstring utf16_string(size, L'\0'); - MultiByteToWideChar(CP_ACP, 0, mb_string.c_str(), mb_string.size(), &*utf16_string.begin(), size); + MultiByteToWideChar(CP_ACP, 0, mb_string.data(), mb_string.size(), &*utf16_string.begin(), size); // Convert from utf-16 to utf-8 return std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t>().to_bytes(utf16_string); #endif diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 2c2f67b169..0437f0c7de 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -112,7 +112,7 @@ static bool multiUserAuthorized(std::string strUserPass) static const unsigned int KEY_SIZE = 32; unsigned char out[KEY_SIZE]; - CHMAC_SHA256(reinterpret_cast<const unsigned char*>(strSalt.c_str()), strSalt.size()).Write(reinterpret_cast<const unsigned char*>(strPass.c_str()), strPass.size()).Finalize(out); + CHMAC_SHA256(reinterpret_cast<const unsigned char*>(strSalt.data()), strSalt.size()).Write(reinterpret_cast<const unsigned char*>(strPass.data()), strPass.size()).Finalize(out); std::vector<unsigned char> hexvec(out, out+KEY_SIZE); std::string strHashFromPass = HexStr(hexvec); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 8113777187..d9c7113323 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -15,6 +15,7 @@ #include <sync.h> #include <ui_interface.h> +#include <deque> #include <memory> #include <stdio.h> #include <stdlib.h> diff --git a/src/init.cpp b/src/init.cpp index 035725b090..2abdf7dbc4 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -29,6 +29,7 @@ #include <net_permissions.h> #include <net_processing.h> #include <netbase.h> +#include <node/context.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -84,9 +85,6 @@ static const bool DEFAULT_STOPAFTERBLOCKIMPORT = false; // Dump addresses to banlist.dat every 15 minutes (900s) static constexpr int DUMP_BANS_INTERVAL = 60 * 15; -std::unique_ptr<CConnman> g_connman; -std::unique_ptr<PeerLogicValidation> peerLogic; -std::unique_ptr<BanMan> g_banman; #ifdef WIN32 // Win32 LevelDB doesn't use filedescriptors, and the ones used for @@ -154,7 +152,7 @@ static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; static boost::thread_group threadGroup; static CScheduler scheduler; -void Interrupt() +void Interrupt(NodeContext& node) { InterruptHTTPServer(); InterruptHTTPRPC(); @@ -162,15 +160,15 @@ void Interrupt() InterruptREST(); InterruptTorControl(); InterruptMapPort(); - if (g_connman) - g_connman->Interrupt(); + if (node.connman) + node.connman->Interrupt(); if (g_txindex) { g_txindex->Interrupt(); } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); } -void Shutdown(InitInterfaces& interfaces) +void Shutdown(NodeContext& node) { LogPrintf("%s: In progress...\n", __func__); static CCriticalSection cs_Shutdown; @@ -189,15 +187,15 @@ void Shutdown(InitInterfaces& interfaces) StopREST(); StopRPC(); StopHTTPServer(); - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->flush(); } StopMapPort(); // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. - if (peerLogic) UnregisterValidationInterface(peerLogic.get()); - if (g_connman) g_connman->Stop(); + if (node.peer_logic) UnregisterValidationInterface(node.peer_logic.get()); + if (node.connman) node.connman->Stop(); if (g_txindex) g_txindex->Stop(); ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); @@ -210,9 +208,9 @@ void Shutdown(InitInterfaces& interfaces) // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. - peerLogic.reset(); - g_connman.reset(); - g_banman.reset(); + node.peer_logic.reset(); + node.connman.reset(); + node.banman.reset(); g_txindex.reset(); DestroyAllBlockFilterIndexes(); @@ -261,7 +259,7 @@ void Shutdown(InitInterfaces& interfaces) } pblocktree.reset(); } - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->stop(); } @@ -280,7 +278,7 @@ void Shutdown(InitInterfaces& interfaces) } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } - interfaces.chain_clients.clear(); + node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); GetMainSignals().UnregisterWithMempoolSignals(mempool); @@ -354,7 +352,7 @@ void SetupServerArgs() std::vector<std::string> hidden_args = { "-dbcrashratio", "-forcecompactdb", // GUI args. These will be overwritten by SetupUIArgs for the GUI - "-allowselfsignedrootcertificates", "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-rootcertificates=<file>", "-splash", "-uiplatform"}; + "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-splash", "-uiplatform"}; gArgs.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM @@ -374,7 +372,7 @@ void SetupServerArgs() gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + gArgs.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -482,7 +480,7 @@ void SetupServerArgs() "(0-4, default: %u)", DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-checkpoints", strprintf("Disable expensive verification for known chain history (default: %u)", DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + gArgs.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block 295000 (default: %u)", DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-deprecatedrpc=<method>", "Allows deprecated RPC method(s) to be used", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -686,20 +684,6 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) LoadGenesisBlock(chainparams); } - // hardcoded $DATADIR/bootstrap.dat - fs::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (fs::exists(pathBootstrap)) { - FILE *file = fsbridge::fopen(pathBootstrap, "rb"); - if (file) { - fs::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; - LogPrintf("Importing bootstrap.dat...\n"); - LoadExternalBlockFile(chainparams, file); - RenameOver(pathBootstrap, pathBootstrapOld); - } else { - LogPrintf("Warning: Could not open bootstrap file %s\n", pathBootstrap.string()); - } - } - // -loadblock= for (const fs::path& path : vImportFiles) { FILE *file = fsbridge::fopen(path, "rb"); @@ -712,7 +696,7 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } // scan for better chains in the block chain database, that are not yet connected in the active best chain - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, chainparams)) { LogPrintf("Failed to connect best block (%s)\n", FormatStateMessage(state)); StartShutdown(); @@ -943,7 +927,7 @@ bool AppInitParameterInteraction() } if (!fs::is_directory(GetBlocksDir())) { - return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.").translated, gArgs.GetArg("-blocksdir", "").c_str())); + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist.").translated, gArgs.GetArg("-blocksdir", ""))); } // parse and validate enabled filter types @@ -1063,15 +1047,6 @@ bool AppInitParameterInteraction() incrementalRelayFee = CFeeRate(n); } - // -par=0 means autodetect, but nScriptCheckThreads==0 means no concurrency - nScriptCheckThreads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); - if (nScriptCheckThreads <= 0) - nScriptCheckThreads += GetNumCores(); - if (nScriptCheckThreads <= 1) - nScriptCheckThreads = 0; - else if (nScriptCheckThreads > MAX_SCRIPTCHECK_THREADS) - nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS; - // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg = gArgs.GetArg("-prune", 0); if (nPruneArg < 0) { @@ -1207,7 +1182,7 @@ bool AppInitLockDataDirectory() return true; } -bool AppInitMain(InitInterfaces& interfaces) +bool AppInitMain(NodeContext& node) { const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization @@ -1258,16 +1233,36 @@ bool AppInitMain(InitInterfaces& interfaces) InitSignatureCache(); InitScriptExecutionCache(); - LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); - if (nScriptCheckThreads) { - for (int i=0; i<nScriptCheckThreads-1; i++) + int script_threads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + if (script_threads <= 0) { + // -par=0 means autodetect (number of cores - 1 script threads) + // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) + script_threads += GetNumCores(); + } + + // Subtract 1 because the main thread counts towards the par threads + script_threads = std::max(script_threads - 1, 0); + + // Number of script-checking threads <= MAX_SCRIPTCHECK_THREADS + script_threads = std::min(script_threads, MAX_SCRIPTCHECK_THREADS); + + LogPrintf("Script verification uses %d additional threads\n", script_threads); + if (script_threads >= 1) { + g_parallel_script_checks = true; + for (int i = 0; i < script_threads; ++i) { threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); + } } // Start the lightweight task scheduler thread CScheduler::Function serviceLoop = std::bind(&CScheduler::serviceQueue, &scheduler); threadGroup.create_thread(std::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); + // Gather some entropy once per minute. + scheduler.scheduleEvery([]{ + RandAddPeriodic(); + }, 60000); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); GetMainSignals().RegisterWithMempoolSignals(mempool); @@ -1275,16 +1270,16 @@ bool AppInitMain(InitInterfaces& interfaces) // according to -wallet and -disablewallet options. This only constructs // the interfaces, it doesn't load wallet data. Wallets actually get loaded // when load() and start() interface methods are called below. - g_wallet_init_interface.Construct(interfaces); + g_wallet_init_interface.Construct(node); /* Register RPC commands regardless of -server setting so they will be * available in the GUI RPC console even if external calls are disabled. */ RegisterAllCoreRPCCommands(tableRPC); - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->registerRpcs(); } - g_rpc_interfaces = &interfaces; + g_rpc_node = &node; #if ENABLE_ZMQ RegisterZMQRPCCommands(tableRPC); #endif @@ -1302,7 +1297,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ********************************************************* Step 5: verify wallet database integrity - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { if (!client->verify()) { return false; } @@ -1314,13 +1309,13 @@ bool AppInitMain(InitInterfaces& interfaces) // is not yet setup and may end up being set up twice if we // need to reindex later. - assert(!g_banman); - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); - assert(!g_connman); - g_connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); + assert(!node.banman); + node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + assert(!node.connman); + node.connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); - peerLogic.reset(new PeerLogicValidation(g_connman.get(), g_banman.get(), scheduler)); - RegisterValidationInterface(peerLogic.get()); + node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), scheduler)); + RegisterValidationInterface(node.peer_logic.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; @@ -1661,7 +1656,7 @@ bool AppInitMain(InitInterfaces& interfaces) } // ********************************************************* Step 9: load wallet - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { if (!client->load()) { return false; } @@ -1765,8 +1760,8 @@ bool AppInitMain(InitInterfaces& interfaces) connOptions.nMaxFeeler = 1; connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; - connOptions.m_banman = g_banman.get(); - connOptions.m_msgproc = peerLogic.get(); + connOptions.m_banman = node.banman.get(); + connOptions.m_msgproc = node.peer_logic.get(); connOptions.nSendBufferMaxSize = 1000*gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000*gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = gArgs.GetArgs("-addnode"); @@ -1806,7 +1801,7 @@ bool AppInitMain(InitInterfaces& interfaces) connOptions.m_specified_outgoing = connect; } } - if (!g_connman->Start(scheduler, connOptions)) { + if (!node.connman->Start(scheduler, connOptions)) { return false; } @@ -1815,12 +1810,13 @@ bool AppInitMain(InitInterfaces& interfaces) SetRPCWarmupFinished(); uiInterface.InitMessage(_("Done loading").translated); - for (const auto& client : interfaces.chain_clients) { + for (const auto& client : node.chain_clients) { client->start(scheduler); } - scheduler.scheduleEvery([]{ - g_banman->DumpBanlist(); + BanMan* banman = node.banman.get(); + scheduler.scheduleEvery([banman]{ + banman->DumpBanlist(); }, DUMP_BANS_INTERVAL * 1000); return true; diff --git a/src/init.h b/src/init.h index 1c59ca069e..ca52dadf08 100644 --- a/src/init.h +++ b/src/init.h @@ -10,26 +10,14 @@ #include <string> #include <util/system.h> -namespace interfaces { -class Chain; -class ChainClient; -} // namespace interfaces - -//! Pointers to interfaces used during init and destroyed on shutdown. -struct InitInterfaces -{ - std::unique_ptr<interfaces::Chain> chain; - std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; -}; - -namespace boost -{ +struct NodeContext; +namespace boost { class thread_group; } // namespace boost /** Interrupt threads */ -void Interrupt(); -void Shutdown(InitInterfaces& interfaces); +void Interrupt(NodeContext& node); +void Shutdown(NodeContext& node); //!Initialize the logging infrastructure void InitLogging(); //!Parameter interaction: change current parameters depending on various rules @@ -63,7 +51,7 @@ bool AppInitLockDataDirectory(); * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(InitInterfaces& interfaces); +bool AppInitMain(NodeContext& node); /** * Setup the arguments for gArgs diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index b2c20573fb..26856a00d3 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -11,6 +11,7 @@ #include <net.h> #include <net_processing.h> #include <node/coin.h> +#include <node/context.h> #include <node/transaction.h> #include <policy/fees.h> #include <policy/policy.h> @@ -57,12 +58,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> } return nullopt; } - int getBlockDepth(const uint256& hash) override - { - const Optional<int> tip_height = getHeight(); - const Optional<int> height = getBlockHeight(hash); - return tip_height && height ? *tip_height - *height + 1 : 0; - } uint256 getBlockHash(int height) override { LockAssertion lock(::cs_main); @@ -181,11 +176,11 @@ public: const CBlockIndex* index, const std::vector<CTransactionRef>& tx_conflicted) override { - m_notifications->BlockConnected(*block, tx_conflicted); + m_notifications->BlockConnected(*block, tx_conflicted, index->nHeight); } - void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->BlockDisconnected(*block); + m_notifications->BlockDisconnected(*block, index->nHeight); } void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override { @@ -238,6 +233,7 @@ public: class ChainImpl : public Chain { public: + explicit ChainImpl(NodeContext& node) : m_node(node) {} std::unique_ptr<Chain::Lock> lock(bool try_lock) override { auto result = MakeUnique<LockImpl>(::cs_main, "cs_main", __FILE__, __LINE__, try_lock); @@ -286,7 +282,7 @@ public: } bool broadcastTransaction(const CTransactionRef& tx, std::string& err_string, const CAmount& max_tx_fee, bool relay) override { - const TransactionError err = BroadcastTransaction(tx, err_string, max_tx_fee, relay, /*wait_callback*/ false); + const TransactionError err = BroadcastTransaction(m_node, tx, err_string, max_tx_fee, relay, /*wait_callback*/ false); // Chain clients only care about failures to accept the tx to the mempool. Disregard non-mempool related failures. // Note: this will need to be updated if BroadcastTransactions() is updated to return other non-mempool failures // that Chain clients do not need to know about. @@ -351,13 +347,11 @@ public: { return MakeUnique<NotificationsHandlerImpl>(*this, notifications); } - void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) override + void waitForNotificationsIfTipChanged(const uint256& old_tip) override { if (!old_tip.IsNull()) { LOCK(::cs_main); if (old_tip == ::ChainActive().Tip()->GetBlockHash()) return; - CBlockIndex* block = LookupBlockIndex(old_tip); - if (block && block->GetAncestor(::ChainActive().Height()) == ::ChainActive().Tip()) return; } SyncWithValidationInterfaceQueue(); } @@ -378,9 +372,10 @@ public: notifications.TransactionAddedToMempool(entry.GetSharedTx()); } } + NodeContext& m_node; }; } // namespace -std::unique_ptr<Chain> MakeChain() { return MakeUnique<ChainImpl>(); } +std::unique_ptr<Chain> MakeChain(NodeContext& node) { return MakeUnique<ChainImpl>(node); } } // namespace interfaces diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 73a78e21fb..349af152d5 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -18,12 +18,12 @@ class CBlock; class CFeeRate; class CRPCCommand; class CScheduler; -class CValidationState; class Coin; class uint256; enum class RBFTransactionState; struct CBlockLocator; struct FeeCalculation; +struct NodeContext; namespace interfaces { @@ -76,10 +76,6 @@ public: //! included in the current chain. virtual Optional<int> getBlockHeight(const uint256& hash) = 0; - //! Get block depth. Returns 1 for chain tip, 2 for preceding block, and - //! so on. Returns 0 for a block not included in the current chain. - virtual int getBlockDepth(const uint256& hash) = 0; - //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; @@ -226,8 +222,8 @@ public: virtual ~Notifications() {} virtual void TransactionAddedToMempool(const CTransactionRef& tx) {} virtual void TransactionRemovedFromMempool(const CTransactionRef& ptx) {} - virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted) {} - virtual void BlockDisconnected(const CBlock& block) {} + virtual void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& tx_conflicted, int height) {} + virtual void BlockDisconnected(const CBlock& block, int height) {} virtual void UpdatedBlockTip() {} virtual void ChainStateFlushed(const CBlockLocator& locator) {} }; @@ -236,9 +232,8 @@ public: virtual std::unique_ptr<Handler> handleNotifications(Notifications& notifications) = 0; //! Wait for pending notifications to be processed unless block hash points to the current - //! chain tip, or to a possible descendant of the current chain tip that isn't currently - //! connected. - virtual void waitForNotificationsIfNewBlocksConnected(const uint256& old_tip) = 0; + //! chain tip. + virtual void waitForNotificationsIfTipChanged(const uint256& old_tip) = 0; //! Register handler for RPC. Command is not copied, so reference //! needs to remain valid until Handler is disconnected. @@ -291,7 +286,7 @@ public: }; //! Return implementation of Chain interface. -std::unique_ptr<Chain> MakeChain(); +std::unique_ptr<Chain> MakeChain(NodeContext& node); //! Return implementation of ChainClient interface for a wallet client. This //! function will be undefined in builds where ENABLE_WALLET is false. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 227ac9f7b9..1877c92178 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -16,6 +16,7 @@ #include <net_processing.h> #include <netaddress.h> #include <netbase.h> +#include <node/context.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/settings.h> @@ -52,7 +53,6 @@ namespace { class NodeImpl : public Node { public: - NodeImpl() { m_interfaces.chain = MakeChain(); } void initError(const std::string& message) override { InitError(message); } bool parseParameters(int argc, const char* const argv[], std::string& error) override { @@ -75,11 +75,15 @@ public: return AppInitBasicSetup() && AppInitParameterInteraction() && AppInitSanityChecks() && AppInitLockDataDirectory(); } - bool appInitMain() override { return AppInitMain(m_interfaces); } + bool appInitMain() override + { + m_context.chain = MakeChain(m_context); + return AppInitMain(m_context); + } void appShutdown() override { - Interrupt(); - Shutdown(m_interfaces); + Interrupt(m_context); + Shutdown(m_context); } void startShutdown() override { StartShutdown(); } bool shutdownRequested() override { return ShutdownRequested(); } @@ -96,15 +100,15 @@ public: bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { - return g_connman ? g_connman->GetNodeCount(flags) : 0; + return m_context.connman ? m_context.connman->GetNodeCount(flags) : 0; } bool getNodesStats(NodesStats& stats) override { stats.clear(); - if (g_connman) { + if (m_context.connman) { std::vector<CNodeStats> stats_temp; - g_connman->GetNodeStats(stats_temp); + m_context.connman->GetNodeStats(stats_temp); stats.reserve(stats_temp.size()); for (auto& node_stats_temp : stats_temp) { @@ -125,44 +129,44 @@ public: } bool getBanned(banmap_t& banmap) override { - if (g_banman) { - g_banman->GetBanned(banmap); + if (m_context.banman) { + m_context.banman->GetBanned(banmap); return true; } return false; } bool ban(const CNetAddr& net_addr, BanReason reason, int64_t ban_time_offset) override { - if (g_banman) { - g_banman->Ban(net_addr, reason, ban_time_offset); + if (m_context.banman) { + m_context.banman->Ban(net_addr, reason, ban_time_offset); return true; } return false; } bool unban(const CSubNet& ip) override { - if (g_banman) { - g_banman->Unban(ip); + if (m_context.banman) { + m_context.banman->Unban(ip); return true; } return false; } bool disconnect(const CNetAddr& net_addr) override { - if (g_connman) { - return g_connman->DisconnectNode(net_addr); + if (m_context.connman) { + return m_context.connman->DisconnectNode(net_addr); } return false; } bool disconnect(NodeId id) override { - if (g_connman) { - return g_connman->DisconnectNode(id); + if (m_context.connman) { + return m_context.connman->DisconnectNode(id); } return false; } - int64_t getTotalBytesRecv() override { return g_connman ? g_connman->GetTotalBytesRecv() : 0; } - int64_t getTotalBytesSent() override { return g_connman ? g_connman->GetTotalBytesSent() : 0; } + int64_t getTotalBytesRecv() override { return m_context.connman ? m_context.connman->GetTotalBytesRecv() : 0; } + int64_t getTotalBytesSent() override { return m_context.connman ? m_context.connman->GetTotalBytesSent() : 0; } size_t getMempoolSize() override { return ::mempool.size(); } size_t getMempoolDynamicUsage() override { return ::mempool.DynamicMemoryUsage(); } bool getHeaderTip(int& height, int64_t& block_time) override @@ -202,11 +206,11 @@ public: bool getImporting() override { return ::fImporting; } void setNetworkActive(bool active) override { - if (g_connman) { - g_connman->SetNetworkActive(active); + if (m_context.connman) { + m_context.connman->SetNetworkActive(active); } } - bool getNetworkActive() override { return g_connman && g_connman->GetNetworkActive(); } + bool getNetworkActive() override { return m_context.connman && m_context.connman->GetNetworkActive(); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) override { FeeCalculation fee_calc; @@ -255,12 +259,12 @@ public: } std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::vector<std::string>& warnings) override { - return MakeWallet(LoadWallet(*m_interfaces.chain, name, error, warnings)); + return MakeWallet(LoadWallet(*m_context.chain, name, error, warnings)); } WalletCreationStatus createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::unique_ptr<Wallet>& result) override { std::shared_ptr<CWallet> wallet; - WalletCreationStatus status = CreateWallet(*m_interfaces.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); + WalletCreationStatus status = CreateWallet(*m_context.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); result = MakeWallet(wallet); return status; } @@ -312,10 +316,11 @@ public: return MakeHandler( ::uiInterface.NotifyHeaderTip_connect([fn](bool initial_download, const CBlockIndex* block) { fn(initial_download, block->nHeight, block->GetBlockTime(), - GuessVerificationProgress(Params().TxData(), block)); + /* verification progress is unused when a header was received */ 0); })); } - InitInterfaces m_interfaces; + NodeContext* context() override { return &m_context; } + NodeContext m_context; }; } // namespace diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 4ee467014c..adf3de7b07 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H -#include <addrdb.h> // For banmap_t #include <amount.h> // For CAmount #include <net.h> // For CConnman::NumConnections +#include <net_types.h> // For banmap_t #include <netaddress.h> // For Network #include <support/allocators/secure.h> // For SecureString @@ -28,6 +28,7 @@ class RPCTimerInterface; class UniValue; class proxyType; struct CNodeStateStats; +struct NodeContext; enum class WalletCreationStatus; namespace interfaces { @@ -254,6 +255,9 @@ public: using NotifyHeaderTipFn = std::function<void(bool initial_download, int height, int64_t block_time, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) = 0; + + //! Return pointer to internal chain interface, useful for testing. + virtual NodeContext* context() { return nullptr; } }; //! Return implementation of Node interface. diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 2353fe0934..701a748e55 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -5,7 +5,6 @@ #include <interfaces/wallet.h> #include <amount.h> -#include <consensus/validation.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <policy/fees.h> @@ -32,7 +31,7 @@ namespace interfaces { namespace { //! Construct wallet tx struct. -WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, const CWalletTx& wtx) +WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) { WalletTx result; result.tx = wtx.tx; @@ -47,10 +46,10 @@ WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, co result.txout_is_mine.emplace_back(wallet.IsMine(txout)); result.txout_address.emplace_back(); result.txout_address_is_mine.emplace_back(ExtractDestination(txout.scriptPubKey, result.txout_address.back()) ? - IsMine(wallet, result.txout_address.back()) : + wallet.IsMine(result.txout_address.back()) : ISMINE_NO); } - result.credit = wtx.GetCredit(locked_chain, ISMINE_ALL); + result.credit = wtx.GetCredit(ISMINE_ALL); result.debit = wtx.GetDebit(ISMINE_ALL); result.change = wtx.GetChange(); result.time = wtx.GetTxTime(); @@ -64,21 +63,20 @@ WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const C { WalletTxStatus result; result.block_height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock).get_value_or(std::numeric_limits<int>::max()); - result.blocks_to_maturity = wtx.GetBlocksToMaturity(locked_chain); - result.depth_in_main_chain = wtx.GetDepthInMainChain(locked_chain); + result.blocks_to_maturity = wtx.GetBlocksToMaturity(); + result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; result.is_final = locked_chain.checkFinalTx(*wtx.tx); result.is_trusted = wtx.IsTrusted(locked_chain); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); - result.is_in_main_chain = wtx.IsInMainChain(locked_chain); + result.is_in_main_chain = wtx.IsInMainChain(); return result; } //! Construct wallet TxOut struct. -WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, - CWallet& wallet, +WalletTxOut MakeWalletTxOut(CWallet& wallet, const CWalletTx& wtx, int n, int depth) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) @@ -87,7 +85,7 @@ WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, result.txout = wtx.tx->vout[n]; result.time = wtx.GetTxTime(); result.depth_in_main_chain = depth; - result.is_spent = wallet.IsSpent(locked_chain, wtx.GetHash(), n); + result.is_spent = wallet.IsSpent(wtx.GetHash(), n); return result; } @@ -118,10 +116,17 @@ public: std::string error; return m_wallet->GetNewDestination(type, label, dest, error); } - bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet->GetPubKey(address, pub_key); } - bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet->GetKey(address, key); } - bool isSpendable(const CTxDestination& dest) override { return IsMine(*m_wallet, dest) & ISMINE_SPENDABLE; } - bool haveWatchOnly() override { return m_wallet->HaveWatchOnly(); }; + bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet->GetLegacyScriptPubKeyMan()->GetPubKey(address, pub_key); } + bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet->GetLegacyScriptPubKeyMan()->GetKey(address, key); } + bool isSpendable(const CTxDestination& dest) override { return m_wallet->IsMine(dest) & ISMINE_SPENDABLE; } + bool haveWatchOnly() override + { + auto spk_man = m_wallet->GetLegacyScriptPubKeyMan(); + if (spk_man) { + return spk_man->HaveWatchOnly(); + } + return false; + }; bool setAddressBook(const CTxDestination& dest, const std::string& name, const std::string& purpose) override { return m_wallet->SetAddressBook(dest, name, purpose); @@ -144,7 +149,7 @@ public: *name = it->second.name; } if (is_mine) { - *is_mine = IsMine(*m_wallet, dest); + *is_mine = m_wallet->IsMine(dest); } if (purpose) { *purpose = it->second.purpose; @@ -156,20 +161,22 @@ public: LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; for (const auto& item : m_wallet->mapAddressBook) { - result.emplace_back(item.first, IsMine(*m_wallet, item.first), item.second.name, item.second.purpose); + result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.name, item.second.purpose); } return result; } - void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->LearnRelatedScripts(key, type); } + void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->GetLegacyScriptPubKeyMan()->LearnRelatedScripts(key, type); } bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { LOCK(m_wallet->cs_wallet); - return m_wallet->AddDestData(dest, key, value); + WalletBatch batch{m_wallet->GetDatabase()}; + return m_wallet->AddDestData(batch, dest, key, value); } bool eraseDestData(const CTxDestination& dest, const std::string& key) override { LOCK(m_wallet->cs_wallet); - return m_wallet->EraseDestData(dest, key); + WalletBatch batch{m_wallet->GetDatabase()}; + return m_wallet->EraseDestData(batch, dest, key); } std::vector<std::string> getDestValues(const std::string& prefix) override { @@ -216,26 +223,20 @@ public: } return tx; } - bool commitTransaction(CTransactionRef tx, + void commitTransaction(CTransactionRef tx, WalletValueMap value_map, - WalletOrderForm order_form, - std::string& reject_reason) override + WalletOrderForm order_form) override { auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); - CValidationState state; - if (!m_wallet->CommitTransaction(std::move(tx), std::move(value_map), std::move(order_form), state)) { - reject_reason = state.GetRejectReason(); - return false; - } - return true; + m_wallet->CommitTransaction(std::move(tx), std::move(value_map), std::move(order_form)); } bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet->TransactionCanBeAbandoned(txid); } bool abandonTransaction(const uint256& txid) override { auto locked_chain = m_wallet->chain().lock(); LOCK(m_wallet->cs_wallet); - return m_wallet->AbandonTransaction(*locked_chain, txid); + return m_wallet->AbandonTransaction(txid); } bool transactionCanBeBumped(const uint256& txid) override { @@ -282,7 +283,7 @@ public: LOCK(m_wallet->cs_wallet); auto mi = m_wallet->mapWallet.find(txid); if (mi != m_wallet->mapWallet.end()) { - return MakeWalletTx(*locked_chain, *m_wallet, mi->second); + return MakeWalletTx(*m_wallet, mi->second); } return {}; } @@ -293,7 +294,7 @@ public: std::vector<WalletTx> result; result.reserve(m_wallet->mapWallet.size()); for (const auto& entry : m_wallet->mapWallet) { - result.emplace_back(MakeWalletTx(*locked_chain, *m_wallet, entry.second)); + result.emplace_back(MakeWalletTx(*m_wallet, entry.second)); } return result; } @@ -338,7 +339,7 @@ public: in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; tx_status = MakeWalletTxStatus(*locked_chain, mi->second); - return MakeWalletTx(*locked_chain, *m_wallet, mi->second); + return MakeWalletTx(*m_wallet, mi->second); } return {}; } @@ -349,7 +350,7 @@ public: result.balance = bal.m_mine_trusted; result.unconfirmed_balance = bal.m_mine_untrusted_pending; result.immature_balance = bal.m_mine_immature; - result.have_watch_only = m_wallet->HaveWatchOnly(); + result.have_watch_only = haveWatchOnly(); if (result.have_watch_only) { result.watch_only_balance = bal.m_watchonly_trusted; result.unconfirmed_watch_only_balance = bal.m_watchonly_untrusted_pending; @@ -407,7 +408,7 @@ public: auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*locked_chain, *m_wallet, *coin.tx, coin.i, coin.nDepth)); + MakeWalletTxOut(*m_wallet, *coin.tx, coin.i, coin.nDepth)); } } return result; @@ -422,9 +423,9 @@ public: result.emplace_back(); auto it = m_wallet->mapWallet.find(output.hash); if (it != m_wallet->mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(*locked_chain); + int depth = it->second.GetDepthInMainChain(); if (depth >= 0) { - result.back() = MakeWalletTxOut(*locked_chain, *m_wallet, it->second, output.n, depth); + result.back() = MakeWalletTxOut(*m_wallet, it->second, output.n, depth); } } } @@ -496,7 +497,11 @@ public: : m_chain(chain), m_wallet_filenames(std::move(wallet_filenames)) { } - void registerRpcs() override { return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); } + void registerRpcs() override + { + g_rpc_chain = &m_chain; + return RegisterWalletRPCCommands(m_chain, m_rpc_handlers); + } bool verify() override { return VerifyWallets(m_chain, m_wallet_filenames); } bool load() override { return LoadWallets(m_chain, m_wallet_filenames); } void start(CScheduler& scheduler) override { return StartWallets(scheduler); } diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 89e056b18b..a96b93b4c3 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -141,10 +141,9 @@ public: std::string& fail_reason) = 0; //! Commit transaction. - virtual bool commitTransaction(CTransactionRef tx, + virtual void commitTransaction(CTransactionRef tx, WalletValueMap value_map, - WalletOrderForm order_form, - std::string& reject_reason) = 0; + WalletOrderForm order_form) = 0; //! Return whether transaction can be abandoned. virtual bool transactionCanBeAbandoned(const uint256& txid) = 0; @@ -162,25 +162,6 @@ struct CExtKey { bool Derive(CExtKey& out, unsigned int nChild) const; CExtPubKey Neuter() const; void SetSeed(const unsigned char* seed, unsigned int nSeedLen); - template <typename Stream> - void Serialize(Stream& s) const - { - unsigned int len = BIP32_EXTKEY_SIZE; - ::WriteCompactSize(s, len); - unsigned char code[BIP32_EXTKEY_SIZE]; - Encode(code); - s.write((const char *)&code[0], len); - } - template <typename Stream> - void Unserialize(Stream& s) - { - unsigned int len = ::ReadCompactSize(s); - unsigned char code[BIP32_EXTKEY_SIZE]; - if (len != BIP32_EXTKEY_SIZE) - throw std::runtime_error("Invalid extended key size\n"); - s.read((char *)&code[0], len); - Decode(code); - } }; /** Initialize the elliptic curve support. May not be called twice without calling ECC_Stop first. */ diff --git a/src/logging.cpp b/src/logging.cpp index 60ab486198..9f6b5ede12 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -67,6 +67,9 @@ bool BCLog::Logger::StartLogging() if (m_print_to_file) FileWriteStr(s, m_fileout); if (m_print_to_console) fwrite(s.data(), 1, s.size(), stdout); + for (const auto& cb : m_print_callbacks) { + cb(s); + } m_msgs_before_open.pop_front(); } @@ -81,6 +84,7 @@ void BCLog::Logger::DisconnectTestLogger() m_buffering = true; if (m_fileout != nullptr) fclose(m_fileout); m_fileout = nullptr; + m_print_callbacks.clear(); } void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) @@ -91,7 +95,15 @@ void BCLog::Logger::EnableCategory(BCLog::LogFlags flag) bool BCLog::Logger::EnableCategory(const std::string& str) { BCLog::LogFlags flag; - if (!GetLogCategory(flag, str)) return false; + if (!GetLogCategory(flag, str)) { + if (str == "db") { + // DEPRECATION: Added in 0.20, should start returning an error in 0.21 + LogPrintf("Warning: logging category 'db' is deprecated, use 'walletdb' instead\n"); + EnableCategory(BCLog::WALLETDB); + return true; + } + return false; + } EnableCategory(flag); return true; } @@ -135,7 +147,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::HTTP, "http"}, {BCLog::BENCH, "bench"}, {BCLog::ZMQ, "zmq"}, - {BCLog::DB, "db"}, + {BCLog::WALLETDB, "walletdb"}, {BCLog::RPC, "rpc"}, {BCLog::ESTIMATEFEE, "estimatefee"}, {BCLog::ADDRMAN, "addrman"}, @@ -270,6 +282,9 @@ void BCLog::Logger::LogPrintStr(const std::string& str) fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } + for (const auto& cb : m_print_callbacks) { + cb(str_prefixed); + } if (m_print_to_file) { assert(m_fileout != nullptr); diff --git a/src/logging.h b/src/logging.h index 75cd5353c0..a2caef51a8 100644 --- a/src/logging.h +++ b/src/logging.h @@ -39,7 +39,7 @@ namespace BCLog { HTTP = (1 << 3), BENCH = (1 << 4), ZMQ = (1 << 5), - DB = (1 << 6), + WALLETDB = (1 << 6), RPC = (1 << 7), ESTIMATEFEE = (1 << 8), ADDRMAN = (1 << 9), @@ -77,6 +77,9 @@ namespace BCLog { std::string LogTimestampStr(const std::string& str); + /** Slots that connect to the print signal */ + std::list<std::function<void(const std::string&)>> m_print_callbacks /* GUARDED_BY(m_cs) */ {}; + public: bool m_print_to_console = false; bool m_print_to_file = false; @@ -95,7 +98,22 @@ namespace BCLog { bool Enabled() const { std::lock_guard<std::mutex> scoped_lock(m_cs); - return m_buffering || m_print_to_console || m_print_to_file; + return m_buffering || m_print_to_console || m_print_to_file || !m_print_callbacks.empty(); + } + + /** Connect a slot to the print signal and return the connection */ + std::list<std::function<void(const std::string&)>>::iterator PushBackCallback(std::function<void(const std::string&)> fun) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.push_back(std::move(fun)); + return --m_print_callbacks.end(); + } + + /** Delete a connection */ + void DeleteCallback(std::list<std::function<void(const std::string&)>>::iterator it) + { + std::lock_guard<std::mutex> scoped_lock(m_cs); + m_print_callbacks.erase(it); } /** Start logging (and flush all buffered messages) */ @@ -155,12 +173,13 @@ static inline void LogPrintf(const char* fmt, const Args&... args) } } -template <typename... Args> -static inline void LogPrint(const BCLog::LogFlags& category, const Args&... args) -{ - if (LogAcceptCategory((category))) { - LogPrintf(args...); - } -} +// Use a macro instead of a function for conditional logging to prevent +// evaluating arguments when logging for the category is not enabled. +#define LogPrint(category, ...) \ + do { \ + if (LogAcceptCategory((category))) { \ + LogPrintf(__VA_ARGS__); \ + } \ + } while (0) #endif // BITCOIN_LOGGING_H diff --git a/src/logging/timer.h b/src/logging/timer.h new file mode 100644 index 0000000000..34dbb942c5 --- /dev/null +++ b/src/logging/timer.h @@ -0,0 +1,104 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_LOGGING_TIMER_H +#define BITCOIN_LOGGING_TIMER_H + +#include <logging.h> +#include <util/macros.h> +#include <util/time.h> + +#include <chrono> +#include <string> + + +namespace BCLog { + +//! RAII-style object that outputs timing information to logs. +template <typename TimeType> +class Timer +{ +public: + //! If log_category is left as the default, end_msg will log unconditionally + //! (instead of being filtered by category). + Timer( + std::string prefix, + std::string end_msg, + BCLog::LogFlags log_category = BCLog::LogFlags::ALL) : + m_prefix(std::move(prefix)), + m_title(std::move(end_msg)), + m_log_category(log_category) + { + this->Log(strprintf("%s started", m_title)); + m_start_t = GetTime<std::chrono::microseconds>(); + } + + ~Timer() + { + this->Log(strprintf("%s completed", m_title)); + } + + void Log(const std::string& msg) + { + const std::string full_msg = this->LogMsg(msg); + + if (m_log_category == BCLog::LogFlags::ALL) { + LogPrintf("%s\n", full_msg); + } else { + LogPrint(m_log_category, "%s\n", full_msg); + } + } + + std::string LogMsg(const std::string& msg) + { + const auto end_time = GetTime<std::chrono::microseconds>() - m_start_t; + if (m_start_t.count() <= 0) { + return strprintf("%s: %s", m_prefix, msg); + } + + std::string units = ""; + float divisor = 1; + + if (std::is_same<TimeType, std::chrono::microseconds>::value) { + units = "μs"; + } else if (std::is_same<TimeType, std::chrono::milliseconds>::value) { + units = "ms"; + divisor = 1000.; + } else if (std::is_same<TimeType, std::chrono::seconds>::value) { + units = "s"; + divisor = 1000. * 1000.; + } + + const float time_ms = end_time.count() / divisor; + return strprintf("%s: %s (%.2f%s)", m_prefix, msg, time_ms, units); + } + +private: + std::chrono::microseconds m_start_t{}; + + //! Log prefix; usually the name of the function this was created in. + const std::string m_prefix{}; + + //! A descriptive message of what is being timed. + const std::string m_title{}; + + //! Forwarded on to LogPrint if specified - has the effect of only + //! outputing the timing log when a particular debug= category is specified. + const BCLog::LogFlags m_log_category{}; + +}; + +} // namespace BCLog + + +#define LOG_TIME_MICROS(end_msg, ...) \ + BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_MILLIS(end_msg, ...) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) +#define LOG_TIME_SECONDS(end_msg, ...) \ + BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, ## __VA_ARGS__) + + +#endif // BITCOIN_LOGGING_TIMER_H diff --git a/src/miner.cpp b/src/miner.cpp index 4f51be8a08..1c9174ee07 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -162,7 +162,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblock->nNonce = 0; pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); - CValidationState state; + BlockValidationState state; if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, FormatStateMessage(state))); } diff --git a/src/net.cpp b/src/net.cpp index c1c70facf4..84692d2a79 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -567,42 +567,28 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete nLastRecv = nTimeMicros / 1000000; nRecvBytes += nBytes; while (nBytes > 0) { - - // get current incomplete message, or create a new one - if (vRecvMsg.empty() || - vRecvMsg.back().complete()) - vRecvMsg.push_back(CNetMessage(Params().MessageStart(), SER_NETWORK, INIT_PROTO_VERSION)); - - CNetMessage& msg = vRecvMsg.back(); - // absorb network data - int handled; - if (!msg.in_data) - handled = msg.readHeader(pch, nBytes); - else - handled = msg.readData(pch, nBytes); - - if (handled < 0) - return false; - - if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); - return false; - } + int handled = m_deserializer->Read(pch, nBytes); + if (handled < 0) return false; pch += handled; nBytes -= handled; - if (msg.complete()) { + if (m_deserializer->Complete()) { + // decompose a transport agnostic CNetMessage from the deserializer + CNetMessage msg = m_deserializer->GetMessage(Params().MessageStart(), nTimeMicros); + //store received bytes per message command //to prevent a memory DOS, only allow valid commands - mapMsgCmdSize::iterator i = mapRecvBytesPerMsgCmd.find(msg.hdr.pchCommand); + mapMsgCmdSize::iterator i = mapRecvBytesPerMsgCmd.find(msg.m_command); if (i == mapRecvBytesPerMsgCmd.end()) i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); assert(i != mapRecvBytesPerMsgCmd.end()); - i->second += msg.hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + i->second += msg.m_raw_message_size; + + // push the message to the process queue, + vRecvMsg.push_back(std::move(msg)); - msg.nTime = nTimeMicros; complete = true; } } @@ -636,8 +622,7 @@ int CNode::GetSendVersion() const return nSendVersion; } - -int CNetMessage::readHeader(const char *pch, unsigned int nBytes) +int V1TransportDeserializer::readHeader(const char *pch, unsigned int nBytes) { // copy data to temporary parsing buffer unsigned int nRemaining = 24 - nHdrPos; @@ -658,9 +643,10 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) return -1; } - // reject messages larger than MAX_SIZE - if (hdr.nMessageSize > MAX_SIZE) + // reject messages larger than MAX_SIZE or MAX_PROTOCOL_MESSAGE_LENGTH + if (hdr.nMessageSize > MAX_SIZE || hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { return -1; + } // switch state to reading message data in_data = true; @@ -668,7 +654,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) return nCopy; } -int CNetMessage::readData(const char *pch, unsigned int nBytes) +int V1TransportDeserializer::readData(const char *pch, unsigned int nBytes) { unsigned int nRemaining = hdr.nMessageSize - nDataPos; unsigned int nCopy = std::min(nRemaining, nBytes); @@ -685,14 +671,44 @@ int CNetMessage::readData(const char *pch, unsigned int nBytes) return nCopy; } -const uint256& CNetMessage::GetMessageHash() const +const uint256& V1TransportDeserializer::GetMessageHash() const { - assert(complete()); + assert(Complete()); if (data_hash.IsNull()) hasher.Finalize(data_hash.begin()); return data_hash; } +CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) { + // decompose a single CNetMessage from the TransportDeserializer + CNetMessage msg(std::move(vRecv)); + + // store state about valid header, netmagic and checksum + msg.m_valid_header = hdr.IsValid(message_start); + msg.m_valid_netmagic = (memcmp(hdr.pchMessageStart, message_start, CMessageHeader::MESSAGE_START_SIZE) == 0); + uint256 hash = GetMessageHash(); + + // store command string, payload size + msg.m_command = hdr.GetCommand(); + msg.m_message_size = hdr.nMessageSize; + msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + + msg.m_valid_checksum = (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) == 0); + if (!msg.m_valid_checksum) { + LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s\n", + SanitizeString(msg.m_command), msg.m_message_size, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + } + + // store receive time + msg.m_time = time; + + // reset the network deserializer (prepare for the next message) + Reset(); + return msg; +} + size_t CConnman::SocketSendData(CNode *pnode) const EXCLUSIVE_LOCKS_REQUIRED(pnode->cs_vSend) { auto it = pnode->vSendMsg.begin(); @@ -1344,9 +1360,9 @@ void CConnman::SocketHandler() size_t nSizeAdded = 0; auto it(pnode->vRecvMsg.begin()); for (; it != pnode->vRecvMsg.end(); ++it) { - if (!it->complete()) - break; - nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; + // vRecvMsg contains only completed CNetMessage + // the single possible partially deserialized message are held by TransportDeserializer + nSizeAdded += it->m_raw_message_size; } { LOCK(pnode->cs_vProcessMsg); @@ -1361,7 +1377,7 @@ void CConnman::SocketHandler() { // socket closed gracefully if (!pnode->fDisconnect) { - LogPrint(BCLog::NET, "socket closed\n"); + LogPrint(BCLog::NET, "socket closed for peer=%d\n", pnode->GetId()); } pnode->CloseSocketDisconnect(); } @@ -1371,8 +1387,9 @@ void CConnman::SocketHandler() int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { - if (!pnode->fDisconnect) - LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); + if (!pnode->fDisconnect) { + LogPrint(BCLog::NET, "socket recv error for peer=%d: %s\n", pnode->GetId(), NetworkErrorString(nErr)); + } pnode->CloseSocketDisconnect(); } } @@ -1457,7 +1474,7 @@ static void ThreadMapPort() if (externalIPAddress[0]) { CNetAddr resolved; if (LookupHost(externalIPAddress, resolved, false)) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString().c_str()); + LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToString()); AddLocal(resolved, LOCAL_UPNP); } } else { @@ -2649,11 +2666,10 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn addrBind(addrBindIn), fInbound(fInboundIn), nKeyedNetGroup(nKeyedNetGroupIn), - addrKnown(5000, 0.001), // Don't relay addr messages to peers that we connect to as block-relay-only // peers (to prevent adversaries from inferring these links from addr // traffic). - m_addr_relay_peer(!block_relay_only), + m_addr_known{block_relay_only ? nullptr : MakeUnique<CRollingBloomFilter>(5000, 0.001)}, id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), @@ -2675,6 +2691,8 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn } else { LogPrint(BCLog::NET, "Added connection peer=%d\n", id); } + + m_deserializer = MakeUnique<V1TransportDeserializer>(V1TransportDeserializer(Params().MessageStart(), SER_NETWORK, INIT_PROTO_VERSION)); } CNode::~CNode() @@ -2691,7 +2709,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->GetId()); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command), nMessageSize, pnode->GetId()); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -480,8 +480,6 @@ private: friend struct CConnmanTest; }; -extern std::unique_ptr<CConnman> g_connman; -extern std::unique_ptr<BanMan> g_banman; void Discover(); void StartMapPort(); void InterruptMapPort(); @@ -609,56 +607,105 @@ public: - +/** Transport protocol agnostic message container. + * Ideally it should only contain receive time, payload, + * command and size. + */ class CNetMessage { +public: + CDataStream m_recv; // received message data + int64_t m_time = 0; // time (in microseconds) of message receipt. + bool m_valid_netmagic = false; + bool m_valid_header = false; + bool m_valid_checksum = false; + uint32_t m_message_size = 0; // size of the payload + uint32_t m_raw_message_size = 0; // used wire size of the message (including header/checksum) + std::string m_command; + + CNetMessage(CDataStream&& recv_in) : m_recv(std::move(recv_in)) {} + + void SetVersion(int nVersionIn) + { + m_recv.SetVersion(nVersionIn); + } +}; + +/** The TransportDeserializer takes care of holding and deserializing the + * network receive buffer. It can deserialize the network buffer into a + * transport protocol agnostic CNetMessage (command & payload) + */ +class TransportDeserializer { +public: + // returns true if the current deserialization is complete + virtual bool Complete() const = 0; + // set the serialization context version + virtual void SetVersion(int version) = 0; + // read and deserialize data + virtual int Read(const char *data, unsigned int bytes) = 0; + // decomposes a message from the context + virtual CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) = 0; + virtual ~TransportDeserializer() {} +}; + +class V1TransportDeserializer final : public TransportDeserializer +{ private: mutable CHash256 hasher; mutable uint256 data_hash; -public: bool in_data; // parsing header (false) or data (true) - CDataStream hdrbuf; // partially received header CMessageHeader hdr; // complete header - unsigned int nHdrPos; - CDataStream vRecv; // received message data + unsigned int nHdrPos; unsigned int nDataPos; - int64_t nTime; // time (in microseconds) of message receipt. + const uint256& GetMessageHash() const; + int readHeader(const char *pch, unsigned int nBytes); + int readData(const char *pch, unsigned int nBytes); - CNetMessage(const CMessageHeader::MessageStartChars& pchMessageStartIn, int nTypeIn, int nVersionIn) : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) { + void Reset() { + vRecv.clear(); + hdrbuf.clear(); hdrbuf.resize(24); in_data = false; nHdrPos = 0; nDataPos = 0; - nTime = 0; + data_hash.SetNull(); + hasher.Reset(); + } + +public: + + V1TransportDeserializer(const CMessageHeader::MessageStartChars& pchMessageStartIn, int nTypeIn, int nVersionIn) : hdrbuf(nTypeIn, nVersionIn), hdr(pchMessageStartIn), vRecv(nTypeIn, nVersionIn) { + Reset(); } - bool complete() const + bool Complete() const override { if (!in_data) return false; return (hdr.nMessageSize == nDataPos); } - - const uint256& GetMessageHash() const; - - void SetVersion(int nVersionIn) + void SetVersion(int nVersionIn) override { hdrbuf.SetVersion(nVersionIn); vRecv.SetVersion(nVersionIn); } - - int readHeader(const char *pch, unsigned int nBytes); - int readData(const char *pch, unsigned int nBytes); + int Read(const char *pch, unsigned int nBytes) override { + int ret = in_data ? readData(pch, nBytes) : readHeader(pch, nBytes); + if (ret < 0) Reset(); + return ret; + } + CNetMessage GetMessage(const CMessageHeader::MessageStartChars& message_start, int64_t time) override; }; - /** Information about a peer */ class CNode { friend class CConnman; public: + std::unique_ptr<TransportDeserializer> m_deserializer; + // socket std::atomic<ServiceFlags> nServices{NODE_NONE}; SOCKET hSocket GUARDED_BY(cs_hSocket); @@ -729,13 +776,12 @@ public: // flood relay std::vector<CAddress> vAddrToSend; - CRollingBloomFilter addrKnown; + const std::unique_ptr<CRollingBloomFilter> m_addr_known; bool fGetAddr{false}; int64_t nNextAddrSend GUARDED_BY(cs_sendProcessing){0}; int64_t nNextLocalAddrSend GUARDED_BY(cs_sendProcessing){0}; - const bool m_addr_relay_peer; - bool IsAddrRelayPeer() const { return m_addr_relay_peer; } + bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } // List of block ids we still have announce. // There is no final sorting before sending, as they are always sent immediately @@ -762,7 +808,7 @@ public: bool fSendMempool GUARDED_BY(cs_tx_inventory){false}; // Last time a "MEMPOOL" request was serviced. std::atomic<std::chrono::seconds> m_last_mempool_req{std::chrono::seconds{0}}; - int64_t nNextInvSend{0}; + std::chrono::microseconds nNextInvSend{0}; CCriticalSection cs_feeFilter; // Minimum fee rate with which to filter inv's to this node @@ -884,7 +930,8 @@ public: void AddAddressKnown(const CAddress& _addr) { - addrKnown.insert(_addr.GetKey()); + assert(m_addr_known); + m_addr_known->insert(_addr.GetKey()); } void PushAddress(const CAddress& _addr, FastRandomContext &insecure_rand) @@ -892,7 +939,8 @@ public: // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. - if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { + assert(m_addr_known); + if (_addr.IsValid() && !m_addr_known->contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { @@ -943,11 +991,13 @@ public: void MaybeSetAddrName(const std::string& addrNameIn); }; - - - - /** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ int64_t PoissonNextSend(int64_t now, int average_interval_seconds); +/** Wrapper to return mockable type */ +inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, std::chrono::seconds average_interval) +{ + return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; +} + #endif // BITCOIN_NET_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 43de12649e..f42a26ca3e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -38,6 +38,8 @@ static constexpr int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; +/** How long to cache transactions in mapRelay for normal relay */ +static constexpr std::chrono::seconds RELAY_TX_CACHE_TIME{15 * 60}; /** Headers download timeout expressed in microseconds * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes @@ -116,8 +118,8 @@ namespace { int nSyncStarted GUARDED_BY(cs_main) = 0; /** - * Sources of received blocks, saved to be able to send them reject - * messages or ban them when processing happens afterwards. + * Sources of received blocks, saved to be able punish them when processing + * happens afterwards. * Set mapBlockSource[hash].second to false if the node should not be * punished if the block is invalid. */ @@ -980,14 +982,12 @@ void Misbehaving(NodeId pnode, int howmuch, const std::string& message) EXCLUSIV * banning/disconnecting us. We use this to determine which unaccepted * transactions from a whitelisted peer that we can safely relay. */ -static bool TxRelayMayResultInDisconnect(const CValidationState& state) -{ - assert(IsTransactionReason(state.GetReason())); - return state.GetReason() == ValidationInvalidReason::CONSENSUS; +static bool TxRelayMayResultInDisconnect(const TxValidationState& state) { + return state.GetResult() == TxValidationResult::TX_CONSENSUS; } /** - * Potentially ban a node based on the contents of a CValidationState object + * Potentially ban a node based on the contents of a BlockValidationState object * * @param[in] via_compact_block: this bool is passed in because net_processing should * punish peers differently depending on whether the data was provided in a compact @@ -995,23 +995,21 @@ static bool TxRelayMayResultInDisconnect(const CValidationState& state) * txs, the peer should not be punished. See BIP 152. * * @return Returns true if the peer was punished (probably disconnected) - * - * Changes here may need to be reflected in TxRelayMayResultInDisconnect(). */ -static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool via_compact_block, const std::string& message = "") { - switch (state.GetReason()) { - case ValidationInvalidReason::NONE: +static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, bool via_compact_block, const std::string& message = "") { + switch (state.GetResult()) { + case BlockValidationResult::BLOCK_RESULT_UNSET: break; // The node is providing invalid data: - case ValidationInvalidReason::CONSENSUS: - case ValidationInvalidReason::BLOCK_MUTATED: + case BlockValidationResult::BLOCK_CONSENSUS: + case BlockValidationResult::BLOCK_MUTATED: if (!via_compact_block) { LOCK(cs_main); Misbehaving(nodeid, 100, message); return true; } break; - case ValidationInvalidReason::CACHED_INVALID: + case BlockValidationResult::BLOCK_CACHED_INVALID: { LOCK(cs_main); CNodeState *node_state = State(nodeid); @@ -1027,30 +1025,24 @@ static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool v } break; } - case ValidationInvalidReason::BLOCK_INVALID_HEADER: - case ValidationInvalidReason::BLOCK_CHECKPOINT: - case ValidationInvalidReason::BLOCK_INVALID_PREV: + case BlockValidationResult::BLOCK_INVALID_HEADER: + case BlockValidationResult::BLOCK_CHECKPOINT: + case BlockValidationResult::BLOCK_INVALID_PREV: { LOCK(cs_main); Misbehaving(nodeid, 100, message); } return true; // Conflicting (but not necessarily invalid) data or different policy: - case ValidationInvalidReason::BLOCK_MISSING_PREV: + case BlockValidationResult::BLOCK_MISSING_PREV: { // TODO: Handle this much more gracefully (10 DoS points is super arbitrary) LOCK(cs_main); Misbehaving(nodeid, 10, message); } return true; - case ValidationInvalidReason::RECENT_CONSENSUS_CHANGE: - case ValidationInvalidReason::BLOCK_TIME_FUTURE: - case ValidationInvalidReason::TX_NOT_STANDARD: - case ValidationInvalidReason::TX_MISSING_INPUTS: - case ValidationInvalidReason::TX_PREMATURE_SPEND: - case ValidationInvalidReason::TX_WITNESS_MUTATED: - case ValidationInvalidReason::TX_CONFLICT: - case ValidationInvalidReason::TX_MEMPOOL_POLICY: + case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: + case BlockValidationResult::BLOCK_TIME_FUTURE: break; } if (message != "") { @@ -1059,6 +1051,39 @@ static bool MaybePunishNode(NodeId nodeid, const CValidationState& state, bool v return false; } +/** + * Potentially ban a node based on the contents of a TxValidationState object + * + * @return Returns true if the peer was punished (probably disconnected) + * + * Changes here may need to be reflected in TxRelayMayResultInDisconnect(). + */ +static bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = "") { + switch (state.GetResult()) { + case TxValidationResult::TX_RESULT_UNSET: + break; + // The node is providing invalid data: + case TxValidationResult::TX_CONSENSUS: + { + LOCK(cs_main); + Misbehaving(nodeid, 100, message); + return true; + } + // Conflicting (but not necessarily invalid) data or different policy: + case TxValidationResult::TX_RECENT_CONSENSUS_CHANGE: + case TxValidationResult::TX_NOT_STANDARD: + case TxValidationResult::TX_MISSING_INPUTS: + case TxValidationResult::TX_PREMATURE_SPEND: + case TxValidationResult::TX_WITNESS_MUTATED: + case TxValidationResult::TX_CONFLICT: + case TxValidationResult::TX_MEMPOOL_POLICY: + break; + } + if (message != "") { + LogPrint(BCLog::NET, "peer=%d: %s\n", nodeid, message); + } + return false; +} @@ -1227,17 +1252,18 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB * Handle invalid block rejection and consequent peer banning, maintain which * peers announce compact blocks. */ -void PeerLogicValidation::BlockChecked(const CBlock& block, const CValidationState& state) { +void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidationState& state) { LOCK(cs_main); const uint256 hash(block.GetHash()); std::map<uint256, std::pair<NodeId, bool>>::iterator it = mapBlockSource.find(hash); - if (state.IsInvalid()) { - // Don't send reject message with code 0 or an internal reject code. - if (it != mapBlockSource.end() && State(it->second.first) && state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) { - MaybePunishNode(/*nodeid=*/ it->second.first, state, /*via_compact_block=*/ !it->second.second); - } + // If the block failed validation, we know where it came from and we're still connected + // to that peer, maybe punish. + if (state.IsInvalid() && + it != mapBlockSource.end() && + State(it->second.first)) { + MaybePunishNodeForBlock(/*nodeid=*/ it->second.first, state, /*via_compact_block=*/ !it->second.second); } // Check that: // 1. The block is valid @@ -1314,7 +1340,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma // Relay to a limited number of other nodes // Use deterministic randomness to send to the same nodes for 24 hours - // at a time so the addrKnowns of the chosen nodes prevent repeats + // at a time so the m_addr_knowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); FastRandomContext insecure_rand; @@ -1375,7 +1401,7 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c } } // release cs_main before calling ActivateBestChain if (need_activate_chain) { - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, Params(), a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", FormatStateMessage(state)); } @@ -1512,6 +1538,10 @@ void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnm // messages from this peer (likely resulting in our peer eventually // disconnecting us). if (pfrom->m_tx_relay != nullptr) { + // mempool entries added before this time have likely expired from mapRelay + const std::chrono::seconds longlived_mempool_time = GetTime<std::chrono::seconds>() - RELAY_TX_CACHE_TIME; + const std::chrono::seconds mempool_req = pfrom->m_tx_relay->m_last_mempool_req.load(); + LOCK(cs_main); while (it != pfrom->vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { @@ -1531,11 +1561,15 @@ void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnm if (mi != mapRelay.end()) { connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); push = true; - } else if (pfrom->m_tx_relay->m_last_mempool_req.load().count()) { + } else { auto txinfo = mempool.info(inv.hash); // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request. - if (txinfo.tx && txinfo.m_time <= pfrom->m_tx_relay->m_last_mempool_req.load()) { + // that TX couldn't have been INVed in reply to a MEMPOOL request, + // or when it's too recent to have expired from mapRelay. + if (txinfo.tx && ( + (mempool_req.count() && txinfo.m_time <= mempool_req) + || (txinfo.m_time <= longlived_mempool_time))) + { connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); push = true; } @@ -1663,11 +1697,10 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve } } - CValidationState state; - CBlockHeader first_invalid_header; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { + BlockValidationState state; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { if (state.IsInvalid()) { - MaybePunishNode(pfrom->GetId(), state, via_compact_block, "invalid header received"); + MaybePunishNodeForBlock(pfrom->GetId(), state, via_compact_block, "invalid header received"); return false; } } @@ -1803,14 +1836,13 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se const CTransactionRef porphanTx = orphan_it->second.tx; const CTransaction& orphanTx = *porphanTx; NodeId fromPeer = orphan_it->second.fromPeer; - bool fMissingInputs2 = false; - // Use a new CValidationState because orphans come from different peers (and we call - // MaybePunishNode based on the source peer from the orphan map, not based on the peer + // Use a new TxValidationState because orphans come from different peers (and we call + // MaybePunishNodeForTx based on the source peer from the orphan map, not based on the peer // that relayed the previous transaction). - CValidationState orphan_state; + TxValidationState orphan_state; if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &fMissingInputs2, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanHash, *connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { @@ -1823,10 +1855,10 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se } EraseOrphanTx(orphanHash); done = true; - } else if (!fMissingInputs2) { + } else if (orphan_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { if (orphan_state.IsInvalid()) { // Punish peer that gave us an invalid orphan tx - if (MaybePunishNode(fromPeer, orphan_state, /*via_compact_block*/ false)) { + if (MaybePunishNodeForTx(fromPeer, orphan_state)) { setMisbehaving.insert(fromPeer); } LogPrint(BCLog::MEMPOOL, " invalid orphan tx %s\n", orphanHash.ToString()); @@ -1834,8 +1866,7 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se // Has inputs but not accepted to mempool // Probably non-standard or insufficient fee LogPrint(BCLog::MEMPOOL, " removed orphan tx %s\n", orphanHash.ToString()); - assert(IsTransactionReason(orphan_state.GetReason())); - if (!orphanTx.HasWitness() && orphan_state.GetReason() != ValidationInvalidReason::TX_WITNESS_MUTATED) { + if (!orphanTx.HasWitness() && orphan_state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { // Do not use rejection cache for witness transactions or // witness-stripped transactions, as they can have been malleated. // See https://github.com/bitcoin/bitcoin/issues/8279 for details. @@ -1849,7 +1880,7 @@ void static ProcessOrphanTx(CConnman* connman, std::set<uint256>& orphan_work_se } } -bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc) +bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -2125,7 +2156,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); - if (g_banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them + if (banman->IsBanned(addr)) continue; // Do not process banned addresses beyond remembering we received them bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { @@ -2280,7 +2311,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_most_recent_block); a_recent_block = most_recent_block; } - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, Params(), a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", FormatStateMessage(state)); } @@ -2460,8 +2491,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK2(cs_main, g_cs_orphans); - bool fMissingInputs = false; - CValidationState state; + TxValidationState state; CNodeState* nodestate = State(pfrom->GetId()); nodestate->m_tx_download.m_tx_announced.erase(inv.hash); @@ -2471,7 +2501,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::list<CTransactionRef> lRemovedTxn; if (!AlreadyHave(inv) && - AcceptToMemoryPool(mempool, state, ptx, &fMissingInputs, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { mempool.check(&::ChainstateActive().CoinsTip()); RelayTransaction(tx.GetHash(), *connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { @@ -2493,7 +2523,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Recursively process any orphan transactions that depended on this one ProcessOrphanTx(connman, pfrom->orphan_work_set, lRemovedTxn); } - else if (fMissingInputs) + else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected for (const CTxIn& txin : tx.vin) { @@ -2526,8 +2556,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(tx.GetHash()); } } else { - assert(IsTransactionReason(state.GetReason())); - if (!tx.HasWitness() && state.GetReason() != ValidationInvalidReason::TX_WITNESS_MUTATED) { + if (!tx.HasWitness() && state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { // Do not use rejection cache for witness transactions or // witness-stripped transactions, as they can have been malleated. // See https://github.com/bitcoin/bitcoin/issues/8279 for details. @@ -2582,7 +2611,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); - MaybePunishNode(pfrom->GetId(), state, /*via_compact_block*/ false); + MaybePunishNodeForTx(pfrom->GetId(), state); } return true; } @@ -2616,10 +2645,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } const CBlockIndex *pindex = nullptr; - CValidationState state; + BlockValidationState state; if (!ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { if (state.IsInvalid()) { - MaybePunishNode(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); + MaybePunishNodeForBlock(pfrom->GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return true; } } @@ -2761,7 +2790,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } // cs_main if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, banman, interruptMsgProc); if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be @@ -2859,11 +2888,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // been run). This is handled below, so just treat this as // though the block was successfully read, and rely on the // handling in ProcessNewBlock to ensure the block index is - // updated, reject messages go out, etc. + // updated, etc. MarkBlockAsReceived(resp.blockhash); // it is now an empty pointer fBlockRead = true; - // mapBlockSource is only used for sending reject messages and DoS scores, - // so the race between here and cs_main in ProcessNewBlock is fine. + // mapBlockSource is used for potentially punishing peers and + // updating which peers send us compact blocks, so the race + // between here and cs_main in ProcessNewBlock is fine. // BIP 152 permits peers to relay compact blocks after validating // the header only; we should not punish peers if the block turns // out to be invalid. @@ -2935,8 +2965,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Also always process if we requested the block explicitly, as we may // need it even though it is not a candidate for a new best tip. forceProcessing |= MarkBlockAsReceived(hash); - // mapBlockSource is only used for sending reject messages and DoS scores, - // so the race between here and cs_main in ProcessNewBlock is fine. + // mapBlockSource is only used for punishing peers and setting + // which peers send us compact blocks, so the race between here and + // cs_main in ProcessNewBlock is fine. mapBlockSource.emplace(hash, std::make_pair(pfrom->GetId(), true)); } bool fNewBlock = false; @@ -2977,7 +3008,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::vector<CAddress> vAddr = connman->GetAddresses(); FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { - if (!g_banman->IsBanned(addr)) { + if (!banman->IsBanned(addr)) { pfrom->PushAddress(addr, insecure_rand); } } @@ -3259,41 +3290,37 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return false; // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); - pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; + pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman->GetReceiveFloodSize(); fMoreWork = !pfrom->vProcessMsg.empty(); } CNetMessage& msg(msgs.front()); msg.SetVersion(pfrom->GetRecvVersion()); - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrint(BCLog::NET, "PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + // Check network magic + if (!msg.m_valid_netmagic) { + LogPrint(BCLog::NET, "PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); pfrom->fDisconnect = true; return false; } - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) + // Check header + if (!msg.m_valid_header) { - LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); return fMoreWork; } - std::string strCommand = hdr.GetCommand(); + const std::string& strCommand = msg.m_command; // Message size - unsigned int nMessageSize = hdr.nMessageSize; + unsigned int nMessageSize = msg.m_message_size; // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + CDataStream& vRecv = msg.m_recv; + if (!msg.m_valid_checksum) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR peer=%d\n", __func__, + SanitizeString(strCommand), nMessageSize, pfrom->GetId()); return fMoreWork; } @@ -3301,7 +3328,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.m_time, chainparams, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) @@ -3548,6 +3575,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Address refresh broadcast int64_t nNow = GetTimeMicros(); + auto current_time = GetTime<std::chrono::microseconds>(); + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { AdvertiseLocal(pto); pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); @@ -3560,11 +3589,12 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); + assert(pto->m_addr_known); for (const CAddress& addr : pto->vAddrToSend) { - if (!pto->addrKnown.contains(addr.GetKey())) + if (!pto->m_addr_known->contains(addr.GetKey())) { - pto->addrKnown.insert(addr.GetKey()); + pto->m_addr_known->insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) @@ -3768,13 +3798,13 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(pto->m_tx_relay->cs_tx_inventory); // Check whether periodic sends should happen bool fSendTrickle = pto->HasPermission(PF_NOBAN); - if (pto->m_tx_relay->nNextInvSend < nNow) { + if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; if (pto->fInbound) { - pto->m_tx_relay->nNextInvSend = connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL); + pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; } else { // Use half the delay for outbound peers, as there is less privacy concern for them. - pto->m_tx_relay->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> 1); + pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, std::chrono::seconds{INVENTORY_BROADCAST_INTERVAL >> 1}); } } @@ -3873,7 +3903,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); if (ret.second) { - vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); + vRelayExpiration.push_back(std::make_pair(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret.first)); } } if (vInv.size() == MAX_INV_SZ) { @@ -3889,7 +3919,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling - const auto current_time = GetTime<std::chrono::microseconds>(); + current_time = GetTime<std::chrono::microseconds>(); // nNow is the current system time (GetTimeMicros is not mockable) and // should be replaced by the mockable current_time eventually nNow = GetTimeMicros(); diff --git a/src/net_processing.h b/src/net_processing.h index e8bc3580dd..4adb7d3a21 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -40,7 +40,7 @@ public: /** * Overridden from CValidationInterface. */ - void BlockChecked(const CBlock& block, const CValidationState& state) override; + void BlockChecked(const CBlock& block, const BlockValidationState& state) override; /** * Overridden from CValidationInterface. */ diff --git a/src/net_types.h b/src/net_types.h new file mode 100644 index 0000000000..d55a8cde6c --- /dev/null +++ b/src/net_types.h @@ -0,0 +1,15 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NET_TYPES_H +#define BITCOIN_NET_TYPES_H + +#include <map> + +class CBanEntry; +class CSubNet; + +using banmap_t = std::map<CSubNet, CBanEntry>; + +#endif // BITCOIN_NET_TYPES_H diff --git a/src/netbase.cpp b/src/netbase.cpp index 0148aea428..d1cde8c40f 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -605,7 +605,7 @@ static void LogConnectFailure(bool manual_connection, const char* fmt, const Arg * @param nTimeout Wait this many milliseconds for the connection to be * established. * @param manual_connection Whether or not the connection was manually requested - * (e.g. thru the addnode RPC) + * (e.g. through the addnode RPC) * * @returns Whether or not a connection was successfully made. */ @@ -709,7 +709,7 @@ bool GetProxy(enum Network net, proxyType &proxyInfoOut) { /** * Set the name proxy to use for all connections to nodes specified by a - * hostname. After setting this proxy, connecting to a node sepcified by a + * hostname. After setting this proxy, connecting to a node specified by a * hostname won't result in a local lookup of said hostname, rather, connect to * the node by asking the name proxy for a proxy connection to the hostname, * effectively delegating the hostname lookup to the specified proxy. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 57fa158ad2..a818f06d51 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -14,9 +14,6 @@ #include <map> -#include <boost/thread.hpp> - - static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); @@ -38,6 +35,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); @@ -51,7 +49,6 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { - boost::this_thread::interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { @@ -61,6 +58,7 @@ bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 7c11aab8bd..a19af0fd1b 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -15,16 +15,17 @@ class CCoinsView; struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint64_t nBogoSize{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + CAmount nTotalAmount{0}; + + //! The number of coins contained. + uint64_t coins_count{0}; }; //! Calculate statistics about the unspent transaction output set diff --git a/src/node/context.cpp b/src/node/context.cpp new file mode 100644 index 0000000000..26a01420c8 --- /dev/null +++ b/src/node/context.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/context.h> + +#include <banman.h> +#include <interfaces/chain.h> +#include <net.h> +#include <net_processing.h> + +NodeContext::NodeContext() {} +NodeContext::~NodeContext() {} diff --git a/src/node/context.h b/src/node/context.h new file mode 100644 index 0000000000..2b124af4db --- /dev/null +++ b/src/node/context.h @@ -0,0 +1,44 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_CONTEXT_H +#define BITCOIN_NODE_CONTEXT_H + +#include <memory> +#include <vector> + +class BanMan; +class CConnman; +class PeerLogicValidation; +namespace interfaces { +class Chain; +class ChainClient; +} // namespace interfaces + +//! NodeContext struct containing references to chain state and connection +//! state. +//! +//! This is used by init, rpc, and test code to pass object references around +//! without needing to declare the same variables and parameters repeatedly, or +//! to use globals. More variables could be added to this struct (particularly +//! references to validation and mempool objects) to eliminate use of globals +//! and make code more modular and testable. The struct isn't intended to have +//! any member functions. It should just be a collection of references that can +//! be used without pulling in unwanted dependencies or functionality. +struct NodeContext +{ + std::unique_ptr<CConnman> connman; + std::unique_ptr<PeerLogicValidation> peer_logic; + std::unique_ptr<BanMan> banman; + std::unique_ptr<interfaces::Chain> chain; + std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; + + //! Declare default constructor and destructor that are not inline, so code + //! instantiating the NodeContext struct doesn't need to #include class + //! definitions for all the unique_ptr members. + NodeContext(); + ~NodeContext(); +}; + +#endif // BITCOIN_NODE_CONTEXT_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 7783671a6c..3c0df2b26e 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -6,6 +6,7 @@ #include <consensus/validation.h> #include <net.h> #include <net_processing.h> +#include <node/context.h> #include <util/validation.h> #include <validation.h> #include <validationinterface.h> @@ -13,12 +14,12 @@ #include <future> -TransactionError BroadcastTransaction(const CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback) +TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback) { // BroadcastTransaction can be called by either sendrawtransaction RPC or wallet RPCs. - // g_connman is assigned both before chain clients and before RPC server is accepting calls, - // and reset after chain clients and RPC sever are stopped. g_connman should never be null here. - assert(g_connman); + // node.connman is assigned both before chain clients and before RPC server is accepting calls, + // and reset after chain clients and RPC sever are stopped. node.connman should never be null here. + assert(node.connman); std::promise<void> promise; uint256 hashTx = tx->GetHash(); bool callback_set = false; @@ -30,24 +31,22 @@ TransactionError BroadcastTransaction(const CTransactionRef tx, std::string& err CCoinsViewCache &view = ::ChainstateActive().CoinsTip(); for (size_t o = 0; o < tx->vout.size(); o++) { const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); - // IsSpent doesnt mean the coin is spent, it means the output doesnt' exist. + // IsSpent doesn't mean the coin is spent, it means the output doesn't exist. // So if the output does exist, then this transaction exists in the chain. if (!existingCoin.IsSpent()) return TransactionError::ALREADY_IN_CHAIN; } if (!mempool.exists(hashTx)) { // Transaction is not already in the mempool. Submit it. - CValidationState state; - bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, std::move(tx), &fMissingInputs, + TxValidationState state; + if (!AcceptToMemoryPool(mempool, state, std::move(tx), nullptr /* plTxnReplaced */, false /* bypass_limits */, max_tx_fee)) { + err_string = FormatStateMessage(state); if (state.IsInvalid()) { - err_string = FormatStateMessage(state); - return TransactionError::MEMPOOL_REJECTED; - } else { - if (fMissingInputs) { + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { return TransactionError::MISSING_INPUTS; } - err_string = FormatStateMessage(state); + return TransactionError::MEMPOOL_REJECTED; + } else { return TransactionError::MEMPOOL_ERROR; } } @@ -79,7 +78,7 @@ TransactionError BroadcastTransaction(const CTransactionRef tx, std::string& err } if (relay) { - RelayTransaction(hashTx, *g_connman); + RelayTransaction(hashTx, *node.connman); } return TransactionError::OK; diff --git a/src/node/transaction.h b/src/node/transaction.h index a3e56544a7..35873d8376 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -9,6 +9,8 @@ #include <primitives/transaction.h> #include <util/error.h> +struct NodeContext; + /** * Submit a transaction to the mempool and (optionally) relay it to all P2P peers. * @@ -18,6 +20,7 @@ * NOT be set while cs_main, cs_mempool or cs_wallet are held to avoid * deadlock. * + * @param[in] node reference to node context * @param[in] tx the transaction to broadcast * @param[out] &err_string reference to std::string to fill with error string if available * @param[in] max_tx_fee reject txs with fees higher than this (if 0, accept any fee) @@ -25,6 +28,6 @@ * @param[in] wait_callback, wait until callbacks have been processed to avoid stale result due to a sequentially RPC. * return error */ -NODISCARD TransactionError BroadcastTransaction(CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); +NODISCARD TransactionError BroadcastTransaction(NodeContext& node, CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 0000000000..702a0cbe53 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,50 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include <uint256.h> +#include <serialize.h> + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } + +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/noui.cpp b/src/noui.cpp index 14d6183d24..a5b7a2d591 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -45,7 +45,7 @@ bool noui_ThreadSafeMessageBox(const std::string& message, const std::string& ca if (!fSecure) { LogPrintf("%s%s\n", strCaption, message); } - tfm::format(std::cerr, "%s%s\n", strCaption.c_str(), message.c_str()); + tfm::format(std::cerr, "%s%s\n", strCaption, message); return false; } @@ -66,28 +66,31 @@ void noui_connect() noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessage); } -bool noui_ThreadSafeMessageBoxSuppressed(const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeMessageBoxRedirect(const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -bool noui_ThreadSafeQuestionSuppressed(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) +bool noui_ThreadSafeQuestionRedirect(const std::string& /* ignored interactive message */, const std::string& message, const std::string& caption, unsigned int style) { + LogPrintf("%s: %s\n", caption, message); return false; } -void noui_InitMessageSuppressed(const std::string& message) +void noui_InitMessageRedirect(const std::string& message) { + LogPrintf("init message: %s\n", message); } -void noui_suppress() +void noui_test_redirect() { noui_ThreadSafeMessageBoxConn.disconnect(); noui_ThreadSafeQuestionConn.disconnect(); noui_InitMessageConn.disconnect(); - noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxSuppressed); - noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionSuppressed); - noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageSuppressed); + noui_ThreadSafeMessageBoxConn = uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBoxRedirect); + noui_ThreadSafeQuestionConn = uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestionRedirect); + noui_InitMessageConn = uiInterface.InitMessage_connect(noui_InitMessageRedirect); } void noui_reconnect() @@ -96,4 +99,4 @@ void noui_reconnect() noui_ThreadSafeQuestionConn.disconnect(); noui_InitMessageConn.disconnect(); noui_connect(); -}
\ No newline at end of file +} diff --git a/src/noui.h b/src/noui.h index 854aeeacca..621e9c2798 100644 --- a/src/noui.h +++ b/src/noui.h @@ -17,10 +17,10 @@ void noui_InitMessage(const std::string& message); /** Connect all bitcoind signal handlers */ void noui_connect(); -/** Suppress all bitcoind signal handlers. Used to suppress output during test runs that produce expected errors */ -void noui_suppress(); +/** Redirect all bitcoind signal handlers to LogPrintf. Used to check or suppress output during test runs that produce expected errors */ +void noui_test_redirect(); -/** Reconnects the regular Non-GUI handlers after having used noui_suppress */ +/** Reconnects the regular Non-GUI handlers after having used noui_test_redirect */ void noui_reconnect(); #endif // BITCOIN_NOUI_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 8154bf105e..a66e4464db 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -517,7 +517,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", - hash.ToString().c_str()); + hash.ToString()); return; } diff --git a/src/psbt.h b/src/psbt.h index 802a7c5ba7..6a5c468058 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -387,7 +387,7 @@ struct PSBTOutput /** A version of CTransaction with the PSBT format*/ struct PartiallySignedTransaction { - boost::optional<CMutableTransaction> tx; + Optional<CMutableTransaction> tx; std::vector<PSBTInput> inputs; std::vector<PSBTOutput> outputs; std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; @@ -401,7 +401,6 @@ struct PartiallySignedTransaction bool AddInput(const CTxIn& txin, PSBTInput& psbtin); bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); PartiallySignedTransaction() {} - PartiallySignedTransaction(const PartiallySignedTransaction& psbt_in) : tx(psbt_in.tx), inputs(psbt_in.inputs), outputs(psbt_in.outputs), unknown(psbt_in.unknown) {} explicit PartiallySignedTransaction(const CMutableTransaction& tx); /** * Finds the UTXO for a given input index diff --git a/src/pubkey.h b/src/pubkey.h index 089324ffda..fd815a871b 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -222,31 +222,6 @@ struct CExtPubKey { void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtPubKey& out, unsigned int nChild) const; - - void Serialize(CSizeComputer& s) const - { - // Optimized implementation for ::GetSerializeSize that avoids copying. - s.seek(BIP32_EXTKEY_SIZE + 1); // add one byte for the size (compact int) - } - template <typename Stream> - void Serialize(Stream& s) const - { - unsigned int len = BIP32_EXTKEY_SIZE; - ::WriteCompactSize(s, len); - unsigned char code[BIP32_EXTKEY_SIZE]; - Encode(code); - s.write((const char *)&code[0], len); - } - template <typename Stream> - void Unserialize(Stream& s) - { - unsigned int len = ::ReadCompactSize(s); - unsigned char code[BIP32_EXTKEY_SIZE]; - if (len != BIP32_EXTKEY_SIZE) - throw std::runtime_error("Invalid extended key size\n"); - s.read((char *)&code[0], len); - Decode(code); - } }; /** Users of this module must hold an ECCVerifyHandle. The constructor and diff --git a/src/qt/README.md b/src/qt/README.md index 0eb18f7cd5..30c68db15b 100644 --- a/src/qt/README.md +++ b/src/qt/README.md @@ -50,7 +50,7 @@ Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt ### paymentserver.(h/cpp) -Used to process BIP21 and BIP70 (see https://github.com/bitcoin/bitcoin/pull/11622) payment URI / requests. Also handles URI based application switching (e.g. when following a bitcoin:... link from a browser). +Used to process BIP21 payment URI requests. Also handles URI based application switching (e.g. when following a bitcoin:... link from a browser). ### walletview.(h/cpp) diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index ae11b80347..48201b420e 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -4,9 +4,9 @@ #include <qt/bantablemodel.h> -#include <qt/clientmodel.h> - #include <interfaces/node.h> +#include <net_types.h> // For banmap_t +#include <qt/clientmodel.h> #include <algorithm> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index f548dc9fe3..676c15ea43 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -309,7 +309,7 @@ void BitcoinApplication::requestShutdown() // rescanning a wallet. m_node.startShutdown(); // Unsetting the client model can cause the current thread to wait for node - // to complete an operation, like wait for a RPC execution to complate. + // to complete an operation, like wait for a RPC execution to complete. window->setClientModel(nullptr); pollShutdownTimer->stop(); @@ -337,10 +337,6 @@ void BitcoinApplication::initializeResult(bool success) window->setWalletController(m_wallet_controller); if (paymentServer) { paymentServer->setOptionsModel(optionsModel); -#ifdef ENABLE_BIP70 - PaymentServer::LoadRootCAs(); - connect(m_wallet_controller, &WalletController::coinsSent, paymentServer, &PaymentServer::fetchPaymentACK); -#endif } } #endif // ENABLE_WALLET @@ -396,14 +392,10 @@ WId BitcoinApplication::getMainWinId() const static void SetupUIArgs() { -#if defined(ENABLE_WALLET) && defined(ENABLE_BIP70) - gArgs.AddArg("-allowselfsignedrootcertificates", strprintf("Allow self signed root certificates (default: %u)", DEFAULT_SELFSIGNED_ROOTCERTS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::GUI); -#endif gArgs.AddArg("-choosedatadir", strprintf("Choose data directory on startup (default: %u)", DEFAULT_CHOOSE_DATADIR), ArgsManager::ALLOW_ANY, OptionsCategory::GUI); gArgs.AddArg("-lang=<lang>", "Set language, for example \"de_DE\" (default: system locale)", ArgsManager::ALLOW_ANY, OptionsCategory::GUI); gArgs.AddArg("-min", "Start minimized", ArgsManager::ALLOW_ANY, OptionsCategory::GUI); gArgs.AddArg("-resetguisettings", "Reset all settings changed in the GUI", ArgsManager::ALLOW_ANY, OptionsCategory::GUI); - gArgs.AddArg("-rootcertificates=<file>", "Set SSL root certificates for payment request (default: -system-)", ArgsManager::ALLOW_ANY, OptionsCategory::GUI); gArgs.AddArg("-splash", strprintf("Show splash screen on startup (default: %u)", DEFAULT_SPLASHSCREEN), ArgsManager::ALLOW_ANY, OptionsCategory::GUI); gArgs.AddArg("-uiplatform", strprintf("Select platform to customize UI for (one of windows, macosx, other; default: %s)", BitcoinGUI::DEFAULT_UIPLATFORM), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::GUI); } @@ -438,16 +430,19 @@ int GuiMain(int argc, char* argv[]) BitcoinApplication app(*node); - // Register meta types used for QMetaObject::invokeMethod - qRegisterMetaType< bool* >(); + // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection + qRegisterMetaType<bool*>(); #ifdef ENABLE_WALLET qRegisterMetaType<WalletModel*>(); #endif - // Need to pass name here as CAmount is a typedef (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) - // IMPORTANT if it is no longer a typedef use the normal variant above - qRegisterMetaType< CAmount >("CAmount"); - qRegisterMetaType< std::function<void()> >("std::function<void()>"); + // Register typedefs (see http://qt-project.org/doc/qt-5/qmetatype.html#qRegisterMetaType) + // IMPORTANT: if CAmount is no longer a typedef use the normal variant above (see https://doc.qt.io/qt-5/qmetatype.html#qRegisterMetaType-1) + qRegisterMetaType<CAmount>("CAmount"); + qRegisterMetaType<size_t>("size_t"); + + qRegisterMetaType<std::function<void()>>("std::function<void()>"); qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); + /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: node->setupServerArgs(); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index b280a0c14f..2aeba6d82c 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -330,7 +330,7 @@ void BitcoinGUI::createActions() usedReceivingAddressesAction->setStatusTip(tr("Show the list of used receiving addresses and labels")); openAction = new QAction(tr("Open &URI..."), this); - openAction->setStatusTip(tr("Open a bitcoin: URI or payment request")); + openAction->setStatusTip(tr("Open a bitcoin: URI")); m_open_wallet_action = new QAction(tr("Open Wallet"), this); m_open_wallet_action->setEnabled(false); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 238be08480..5b216b2705 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -19,6 +19,7 @@ #include <stdint.h> #include <QDebug> +#include <QThread> #include <QTimer> static int64_t nLastHeaderTipUpdateNotification = 0; @@ -30,15 +31,26 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO optionsModel(_optionsModel), peerTableModel(nullptr), banTableModel(nullptr), - pollTimer(nullptr) + m_thread(new QThread(this)) { cachedBestHeaderHeight = -1; cachedBestHeaderTime = -1; peerTableModel = new PeerTableModel(m_node, this); banTableModel = new BanTableModel(m_node, this); - pollTimer = new QTimer(this); - connect(pollTimer, &QTimer::timeout, this, &ClientModel::updateTimer); - pollTimer->start(MODEL_UPDATE_DELAY); + + QTimer* timer = new QTimer; + timer->setInterval(MODEL_UPDATE_DELAY); + connect(timer, &QTimer::timeout, [this] { + // no locking required at this point + // the following calls will acquire the required lock + Q_EMIT mempoolSizeChanged(m_node.getMempoolSize(), m_node.getMempoolDynamicUsage()); + Q_EMIT bytesChanged(m_node.getTotalBytesRecv(), m_node.getTotalBytesSent()); + }); + connect(m_thread, &QThread::finished, timer, &QObject::deleteLater); + connect(m_thread, &QThread::started, [timer] { timer->start(); }); + // move timer to thread so that polling doesn't disturb main event loop + timer->moveToThread(m_thread); + m_thread->start(); subscribeToCoreSignals(); } @@ -46,6 +58,9 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO ClientModel::~ClientModel() { unsubscribeFromCoreSignals(); + + m_thread->quit(); + m_thread->wait(); } int ClientModel::getNumConnections(unsigned int flags) const @@ -90,14 +105,6 @@ int64_t ClientModel::getHeaderTipTime() const return cachedBestHeaderTime; } -void ClientModel::updateTimer() -{ - // no locking required at this point - // the following calls will acquire the required lock - Q_EMIT mempoolSizeChanged(m_node.getMempoolSize(), m_node.getMempoolDynamicUsage()); - Q_EMIT bytesChanged(m_node.getTotalBytesRecv(), m_node.getTotalBytesSent()); -} - void ClientModel::updateNumConnections(int numConnections) { Q_EMIT numConnectionsChanged(numConnections); diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 95f4521f06..d3a95d531e 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -90,7 +90,8 @@ private: PeerTableModel *peerTableModel; BanTableModel *banTableModel; - QTimer *pollTimer; + //! A thread to interact with m_node asynchronously + QThread* const m_thread; void subscribeToCoreSignals(); void unsubscribeFromCoreSignals(); @@ -110,7 +111,6 @@ Q_SIGNALS: void showProgress(const QString &title, int nProgress); public Q_SLOTS: - void updateTimer(); void updateNumConnections(int numConnections); void updateNetworkActive(bool networkActive); void updateAlert(); diff --git a/src/qt/forms/openuridialog.ui b/src/qt/forms/openuridialog.ui index 0e1048bc07..2acec314fd 100644 --- a/src/qt/forms/openuridialog.ui +++ b/src/qt/forms/openuridialog.ui @@ -11,17 +11,10 @@ </rect> </property> <property name="windowTitle"> - <string>Open URI</string> + <string>Open bitcoin URI</string> </property> <layout class="QVBoxLayout" name="verticalLayout"> <item> - <widget class="QLabel" name="label_2"> - <property name="text"> - <string>Open payment request from URI or file</string> - </property> - </widget> - </item> - <item> <layout class="QHBoxLayout" name="horizontalLayout"> <item> <widget class="QLabel" name="label"> @@ -33,19 +26,6 @@ <item> <widget class="QValidatedLineEdit" name="uriEdit"/> </item> - <item> - <widget class="QPushButton" name="selectFileButton"> - <property name="toolTip"> - <string>Select payment request file</string> - </property> - <property name="text"> - <string notr="true">…</string> - </property> - <property name="autoDefault"> - <bool>false</bool> - </property> - </widget> - </item> </layout> </item> <item> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 386d559281..7190d59240 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -797,7 +797,7 @@ <item> <widget class="QPushButton" name="buttonMinimizeFee"> <property name="toolTip"> - <string>collapse fee-settings</string> + <string>Hide transaction fee settings</string> </property> <property name="text"> <string>Hide</string> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index 3c699abc6a..843d909f68 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -17,9 +17,6 @@ <bool>false</bool> </property> <widget class="QFrame" name="SendCoins"> - <property name="toolTip"> - <string>This is a normal payment.</string> - </property> <property name="frameShape"> <enum>QFrame::NoFrame</enum> </property> @@ -165,7 +162,11 @@ <item row="2" column="1"> <layout class="QHBoxLayout" name="horizontalLayoutAmount" stretch="0,1,0"> <item> - <widget class="BitcoinAmountField" name="payAmount"/> + <widget class="BitcoinAmountField" name="payAmount"> + <property name="toolTip"> + <string>The amount to send in the selected unit</string> + </property> + </widget> </item> <item> <widget class="QCheckBox" name="checkboxSubtractFeeFromAmount"> diff --git a/src/qt/networkstyle.h b/src/qt/networkstyle.h index bb12dd1b6e..1367261325 100644 --- a/src/qt/networkstyle.h +++ b/src/qt/networkstyle.h @@ -13,7 +13,7 @@ class NetworkStyle { public: - /** Get style associated with provided BIP70 network id, or 0 if not known */ + /** Get style associated with provided network id, or 0 if not known */ static const NetworkStyle* instantiate(const std::string& networkId); const QString &getAppName() const { return appName; } diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 48db95679f..199804f84d 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -39,12 +39,3 @@ void OpenURIDialog::accept() ui->uriEdit->setValid(false); } } - -void OpenURIDialog::on_selectFileButton_clicked() -{ - QString filename = GUIUtil::getOpenFileName(this, tr("Select payment request file to open"), "", "", nullptr); - if(filename.isEmpty()) - return; - QUrl fileUri = QUrl::fromLocalFile(filename); - ui->uriEdit->setText("bitcoin:?r=" + QUrl::toPercentEncoding(fileUri.toString())); -} diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index e94593d5bb..8438f22bd7 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -24,9 +24,6 @@ public: protected Q_SLOTS: void accept(); -private Q_SLOTS: - void on_selectFileButton_clicked(); - private: Ui::OpenURIDialog *ui; }; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d047a82475..d74d0dbfeb 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -18,7 +18,7 @@ #include <netbase.h> #include <txdb.h> // for -dbcache defaults -#include <QNetworkProxy> +#include <QDebug> #include <QSettings> #include <QStringList> @@ -483,24 +483,6 @@ void OptionsModel::setDisplayUnit(const QVariant &value) } } -bool OptionsModel::getProxySettings(QNetworkProxy& proxy) const -{ - // Directly query current base proxy, because - // GUI settings can be overridden with -proxy. - proxyType curProxy; - if (m_node.getProxy(NET_IPV4, curProxy)) { - proxy.setType(QNetworkProxy::Socks5Proxy); - proxy.setHostName(QString::fromStdString(curProxy.proxy.ToStringIP())); - proxy.setPort(curProxy.proxy.GetPort()); - - return true; - } - else - proxy.setType(QNetworkProxy::NoProxy); - - return false; -} - void OptionsModel::setRestartRequired(bool fRequired) { QSettings settings; diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index b1231b7c7d..5791b47f28 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -13,10 +13,6 @@ namespace interfaces { class Node; } -QT_BEGIN_NAMESPACE -class QNetworkProxy; -QT_END_NAMESPACE - extern const char *DEFAULT_GUI_PROXY_HOST; static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; @@ -73,7 +69,6 @@ public: bool getMinimizeOnClose() const { return fMinimizeOnClose; } int getDisplayUnit() const { return nDisplayUnit; } QString getThirdPartyTxUrls() const { return strThirdPartyTxUrls; } - bool getProxySettings(QNetworkProxy& proxy) const; bool getCoinControlFeatures() const { return fCoinControlFeatures; } const QString& getOverriddenByCommandLine() { return strOverriddenByCommandLine; } diff --git a/src/qt/paymentrequest.proto b/src/qt/paymentrequest.proto deleted file mode 100644 index d2721a34bd..0000000000 --- a/src/qt/paymentrequest.proto +++ /dev/null @@ -1,48 +0,0 @@ -// -// Simple Bitcoin Payment Protocol messages -// -// Use fields 100+ for extensions; -// to avoid conflicts, register extensions at: -// https://en.bitcoin.it/wiki/Payment_Request -// - -syntax = "proto2"; - -package payments; -option java_package = "org.bitcoin.protocols.payments"; -option java_outer_classname = "Protos"; - -// Generalized form of "send payment to this/these bitcoin addresses" -message Output { - optional uint64 amount = 1 [default = 0]; // amount is integer-number-of-satoshis - required bytes script = 2; // usually one of the standard Script forms -} -message PaymentDetails { - optional string network = 1 [default = "main"]; // "main" or "test" - repeated Output outputs = 2; // Where payment should be sent - required uint64 time = 3; // Timestamp; when payment request created - optional uint64 expires = 4; // Timestamp; when this request should be considered invalid - optional string memo = 5; // Human-readable description of request for the customer - optional string payment_url = 6; // URL to send Payment and get PaymentACK - optional bytes merchant_data = 7; // Arbitrary data to include in the Payment message -} -message PaymentRequest { - optional uint32 payment_details_version = 1 [default = 1]; - optional string pki_type = 2 [default = "none"]; // none / x509+sha256 / x509+sha1 - optional bytes pki_data = 3; // depends on pki_type - required bytes serialized_payment_details = 4; // PaymentDetails - optional bytes signature = 5; // pki-dependent signature -} -message X509Certificates { - repeated bytes certificate = 1; // DER-encoded X.509 certificate chain -} -message Payment { - optional bytes merchant_data = 1; // From PaymentDetails.merchant_data - repeated bytes transactions = 2; // Signed transactions that satisfy PaymentDetails.outputs - repeated Output refund_to = 3; // Where to send refunds, if a refund is necessary - optional string memo = 4; // Human-readable message for the merchant -} -message PaymentACK { - required Payment payment = 1; // Payment message that triggered this ACK - optional string memo = 2; // human-readable message for customer -} diff --git a/src/qt/paymentrequestplus.cpp b/src/qt/paymentrequestplus.cpp deleted file mode 100644 index b962ab1ef2..0000000000 --- a/src/qt/paymentrequestplus.cpp +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -// -// Wraps dumb protocol buffer paymentRequest -// with some extra methods -// - -#include <qt/paymentrequestplus.h> - -#include <util/system.h> - -#include <stdexcept> - -#include <openssl/x509_vfy.h> - -#include <QDateTime> -#include <QDebug> -#include <QSslCertificate> - -class SSLVerifyError : public std::runtime_error -{ -public: - explicit SSLVerifyError(std::string err) : std::runtime_error(err) { } -}; - -bool PaymentRequestPlus::parse(const QByteArray& data) -{ - bool parseOK = paymentRequest.ParseFromArray(data.data(), data.size()); - if (!parseOK) { - qWarning() << "PaymentRequestPlus::parse: Error parsing payment request"; - return false; - } - if (paymentRequest.payment_details_version() > 1) { - qWarning() << "PaymentRequestPlus::parse: Received up-version payment details, version=" << paymentRequest.payment_details_version(); - return false; - } - - parseOK = details.ParseFromString(paymentRequest.serialized_payment_details()); - if (!parseOK) - { - qWarning() << "PaymentRequestPlus::parse: Error parsing payment details"; - paymentRequest.Clear(); - return false; - } - return true; -} - -bool PaymentRequestPlus::SerializeToString(std::string* output) const -{ - return paymentRequest.SerializeToString(output); -} - -bool PaymentRequestPlus::IsInitialized() const -{ - return paymentRequest.IsInitialized(); -} - -bool PaymentRequestPlus::getMerchant(X509_STORE* certStore, QString& merchant) const -{ - merchant.clear(); - - if (!IsInitialized()) - return false; - - // One day we'll support more PKI types, but just - // x509 for now: - const EVP_MD* digestAlgorithm = nullptr; - if (paymentRequest.pki_type() == "x509+sha256") { - digestAlgorithm = EVP_sha256(); - } - else if (paymentRequest.pki_type() == "x509+sha1") { - digestAlgorithm = EVP_sha1(); - } - else if (paymentRequest.pki_type() == "none") { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: pki_type == none"; - return false; - } - else { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: unknown pki_type " << QString::fromStdString(paymentRequest.pki_type()); - return false; - } - - payments::X509Certificates certChain; - if (!certChain.ParseFromString(paymentRequest.pki_data())) { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: error parsing pki_data"; - return false; - } - - std::vector<X509*> certs; - const QDateTime currentTime = QDateTime::currentDateTime(); - for (int i = 0; i < certChain.certificate_size(); i++) { - QByteArray certData(certChain.certificate(i).data(), certChain.certificate(i).size()); - QSslCertificate qCert(certData, QSsl::Der); - if (currentTime < qCert.effectiveDate() || currentTime > qCert.expiryDate()) { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: certificate expired or not yet active: " << qCert; - return false; - } - if (qCert.isBlacklisted()) { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: certificate blacklisted: " << qCert; - return false; - } - const unsigned char *data = (const unsigned char *)certChain.certificate(i).data(); - X509 *cert = d2i_X509(nullptr, &data, certChain.certificate(i).size()); - if (cert) - certs.push_back(cert); - } - if (certs.empty()) { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: empty certificate chain"; - return false; - } - - // The first cert is the signing cert, the rest are untrusted certs that chain - // to a valid root authority. OpenSSL needs them separately. - STACK_OF(X509) *chain = sk_X509_new_null(); - for (int i = certs.size() - 1; i > 0; i--) { - sk_X509_push(chain, certs[i]); - } - X509 *signing_cert = certs[0]; - - // Now create a "store context", which is a single use object for checking, - // load the signing cert into it and verify. - X509_STORE_CTX *store_ctx = X509_STORE_CTX_new(); - if (!store_ctx) { - qWarning() << "PaymentRequestPlus::getMerchant: Payment request: error creating X509_STORE_CTX"; - return false; - } - - char *website = nullptr; - bool fResult = true; - try - { - if (!X509_STORE_CTX_init(store_ctx, certStore, signing_cert, chain)) - { - int error = X509_STORE_CTX_get_error(store_ctx); - throw SSLVerifyError(X509_verify_cert_error_string(error)); - } - - // Now do the verification! - int result = X509_verify_cert(store_ctx); - if (result != 1) { - int error = X509_STORE_CTX_get_error(store_ctx); - // For testing payment requests, we allow self signed root certs! - if (!(error == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT && gArgs.GetBoolArg("-allowselfsignedrootcertificates", DEFAULT_SELFSIGNED_ROOTCERTS))) { - throw SSLVerifyError(X509_verify_cert_error_string(error)); - } else { - qDebug() << "PaymentRequestPlus::getMerchant: Allowing self signed root certificate, because -allowselfsignedrootcertificates is true."; - } - } - X509_NAME *certname = X509_get_subject_name(signing_cert); - - // Valid cert; check signature: - payments::PaymentRequest rcopy(paymentRequest); // Copy - rcopy.set_signature(std::string("")); - std::string data_to_verify; // Everything but the signature - rcopy.SerializeToString(&data_to_verify); - -#if HAVE_DECL_EVP_MD_CTX_NEW - EVP_MD_CTX *ctx = EVP_MD_CTX_new(); - if (!ctx) throw SSLVerifyError("Error allocating OpenSSL context."); -#else - EVP_MD_CTX _ctx; - EVP_MD_CTX *ctx; - ctx = &_ctx; -#endif - EVP_PKEY *pubkey = X509_get_pubkey(signing_cert); - EVP_MD_CTX_init(ctx); - if (!EVP_VerifyInit_ex(ctx, digestAlgorithm, nullptr) || - !EVP_VerifyUpdate(ctx, data_to_verify.data(), data_to_verify.size()) || - !EVP_VerifyFinal(ctx, (const unsigned char*)paymentRequest.signature().data(), (unsigned int)paymentRequest.signature().size(), pubkey)) { - throw SSLVerifyError("Bad signature, invalid payment request."); - } -#if HAVE_DECL_EVP_MD_CTX_NEW - EVP_MD_CTX_free(ctx); -#endif - - // OpenSSL API for getting human printable strings from certs is baroque. - int textlen = X509_NAME_get_text_by_NID(certname, NID_commonName, nullptr, 0); - website = new char[textlen + 1]; - if (X509_NAME_get_text_by_NID(certname, NID_commonName, website, textlen + 1) == textlen && textlen > 0) { - merchant = website; - } - else { - throw SSLVerifyError("Bad certificate, missing common name."); - } - // TODO: detect EV certificates and set merchant = business name instead of unfriendly NID_commonName ? - } - catch (const SSLVerifyError& err) { - fResult = false; - qWarning() << "PaymentRequestPlus::getMerchant: SSL error: " << err.what(); - } - - delete[] website; - X509_STORE_CTX_free(store_ctx); - for (unsigned int i = 0; i < certs.size(); i++) - X509_free(certs[i]); - - return fResult; -} - -QList<std::pair<CScript,CAmount> > PaymentRequestPlus::getPayTo() const -{ - QList<std::pair<CScript,CAmount> > result; - for (int i = 0; i < details.outputs_size(); i++) - { - const unsigned char* scriptStr = (const unsigned char*)details.outputs(i).script().data(); - CScript s(scriptStr, scriptStr+details.outputs(i).script().size()); - - result.append(std::make_pair(s, details.outputs(i).amount())); - } - return result; -} diff --git a/src/qt/paymentrequestplus.h b/src/qt/paymentrequestplus.h deleted file mode 100644 index 3014628807..0000000000 --- a/src/qt/paymentrequestplus.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_QT_PAYMENTREQUESTPLUS_H -#define BITCOIN_QT_PAYMENTREQUESTPLUS_H - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include <qt/paymentrequest.pb.h> -#pragma GCC diagnostic pop - -#include <amount.h> -#include <script/script.h> - -#include <openssl/x509.h> - -#include <QByteArray> -#include <QList> -#include <QString> - -static const bool DEFAULT_SELFSIGNED_ROOTCERTS = false; - -// -// Wraps dumb protocol buffer paymentRequest -// with extra methods -// - -class PaymentRequestPlus -{ -public: - PaymentRequestPlus() { } - - bool parse(const QByteArray& data); - bool SerializeToString(std::string* output) const; - - bool IsInitialized() const; - // Returns true if merchant's identity is authenticated, and - // returns human-readable merchant identity in merchant - bool getMerchant(X509_STORE* certStore, QString& merchant) const; - - // Returns list of outputs, amount - QList<std::pair<CScript,CAmount> > getPayTo() const; - - const payments::PaymentDetails& getDetails() const { return details; } - -private: - payments::PaymentRequest paymentRequest; - payments::PaymentDetails details; -}; - -#endif // BITCOIN_QT_PAYMENTREQUESTPLUS_H diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 806cc3c41e..6ad219ca2d 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -23,8 +23,6 @@ #include <cstdlib> #include <memory> -#include <openssl/x509_vfy.h> - #include <QApplication> #include <QByteArray> #include <QDataStream> @@ -36,28 +34,11 @@ #include <QList> #include <QLocalServer> #include <QLocalSocket> -#include <QNetworkAccessManager> -#include <QNetworkProxy> -#include <QNetworkReply> -#include <QNetworkRequest> -#include <QSslCertificate> -#include <QSslConfiguration> -#include <QSslError> #include <QStringList> -#include <QTextDocument> #include <QUrlQuery> const int BITCOIN_IPC_CONNECT_TIMEOUT = 1000; // milliseconds const QString BITCOIN_IPC_PREFIX("bitcoin:"); -#ifdef ENABLE_BIP70 -// BIP70 payment protocol messages -const char* BIP70_MESSAGE_PAYMENTACK = "PaymentACK"; -const char* BIP70_MESSAGE_PAYMENTREQUEST = "PaymentRequest"; -// BIP71 payment protocol media types -const char* BIP71_MIMETYPE_PAYMENT = "application/bitcoin-payment"; -const char* BIP71_MIMETYPE_PAYMENTACK = "application/bitcoin-paymentack"; -const char* BIP71_MIMETYPE_PAYMENTREQUEST = "application/bitcoin-paymentrequest"; -#endif // // Create a name that is unique for: @@ -125,32 +106,6 @@ void PaymentServer::ipcParseCommandLine(interfaces::Node& node, int argc, char* } } } -#ifdef ENABLE_BIP70 - else if (QFile::exists(arg)) // Filename - { - if (savedPaymentRequests.contains(arg)) continue; - savedPaymentRequests.insert(arg); - - PaymentRequestPlus request; - if (readPaymentRequestFromFile(arg, request)) - { - if (request.getDetails().network() == "main") - { - node.selectParams(CBaseChainParams::MAIN); - } - else if (request.getDetails().network() == "test") - { - node.selectParams(CBaseChainParams::TESTNET); - } - } - } - else - { - // Printing to debug.log is about the best we can do here, the - // GUI hasn't started yet so we can't pop up a message box. - qWarning() << "PaymentServer::ipcSendCommandLine: Payment request file does not exist: " << arg; - } -#endif } } @@ -198,16 +153,7 @@ PaymentServer::PaymentServer(QObject* parent, bool startLocalServer) : saveURIs(true), uriServer(nullptr), optionsModel(nullptr) -#ifdef ENABLE_BIP70 - ,netManager(nullptr) -#endif { -#ifdef ENABLE_BIP70 - // Verify that the version of the library that we linked against is - // compatible with the version of the headers we compiled against. - GOOGLE_PROTOBUF_VERIFY_VERSION; -#endif - // Install global event filter to catch QFileOpenEvents // on Mac: sent when you click bitcoin: links // other OSes: helpful when dealing with payment request files @@ -230,24 +176,16 @@ PaymentServer::PaymentServer(QObject* parent, bool startLocalServer) : } else { connect(uriServer, &QLocalServer::newConnection, this, &PaymentServer::handleURIConnection); -#ifdef ENABLE_BIP70 - connect(this, &PaymentServer::receivedPaymentACK, this, &PaymentServer::handlePaymentACK); -#endif } } } PaymentServer::~PaymentServer() { -#ifdef ENABLE_BIP70 - google::protobuf::ShutdownProtobufLibrary(); -#endif } // -// OSX-specific way of handling bitcoin: URIs and PaymentRequest mime types. -// Also used by paymentservertests.cpp and when opening a payment request file -// via "Open URI..." menu entry. +// OSX-specific way of handling bitcoin: URIs // bool PaymentServer::eventFilter(QObject *object, QEvent *event) { @@ -266,10 +204,6 @@ bool PaymentServer::eventFilter(QObject *object, QEvent *event) void PaymentServer::uiReady() { -#ifdef ENABLE_BIP70 - initNetManager(); -#endif - saveURIs = false; for (const QString& s : savedPaymentRequests) { @@ -294,48 +228,19 @@ void PaymentServer::handleURIOrFile(const QString& s) else if (s.startsWith(BITCOIN_IPC_PREFIX, Qt::CaseInsensitive)) // bitcoin: URI { QUrlQuery uri((QUrl(s))); -#ifdef ENABLE_BIP70 - if (uri.hasQueryItem("r")) // payment request URI - { - Q_EMIT message(tr("URI handling"), - tr("You are using a BIP70 URL which will be unsupported in the future."), - CClientUIInterface::ICON_WARNING); - QByteArray temp; - temp.append(uri.queryItemValue("r")); - QString decoded = QUrl::fromPercentEncoding(temp); - QUrl fetchUrl(decoded, QUrl::StrictMode); - - if (fetchUrl.isValid()) - { - qDebug() << "PaymentServer::handleURIOrFile: fetchRequest(" << fetchUrl << ")"; - fetchRequest(fetchUrl); - } - else - { - qWarning() << "PaymentServer::handleURIOrFile: Invalid URL: " << fetchUrl; - Q_EMIT message(tr("URI handling"), - tr("Payment request fetch URL is invalid: %1").arg(fetchUrl.toString()), - CClientUIInterface::ICON_WARNING); - } - return; - } - else -#endif // normal URI { SendCoinsRecipient recipient; if (GUIUtil::parseBitcoinURI(s, &recipient)) { if (!IsValidDestinationString(recipient.address.toStdString())) { -#ifndef ENABLE_BIP70 if (uri.hasQueryItem("r")) { // payment request Q_EMIT message(tr("URI handling"), - tr("Cannot process payment request because BIP70 support was not compiled in.")+ + tr("Cannot process payment request because BIP70 is not supported.")+ tr("Due to widespread security flaws in BIP70 it's strongly recommended that any merchant instructions to switch wallets be ignored.")+ tr("If you are receiving this error you should request the merchant provide a BIP21 compatible URI."), CClientUIInterface::ICON_WARNING); } -#endif Q_EMIT message(tr("URI handling"), tr("Invalid payment address %1").arg(recipient.address), CClientUIInterface::MSG_ERROR); } @@ -353,26 +258,11 @@ void PaymentServer::handleURIOrFile(const QString& s) if (QFile::exists(s)) // payment request file { -#ifdef ENABLE_BIP70 - PaymentRequestPlus request; - SendCoinsRecipient recipient; - if (!readPaymentRequestFromFile(s, request)) - { - Q_EMIT message(tr("Payment request file handling"), - tr("Payment request file cannot be read! This can be caused by an invalid payment request file."), - CClientUIInterface::ICON_WARNING); - } - else if (processPaymentRequest(request, recipient)) - Q_EMIT receivedPaymentRequest(recipient); - - return; -#else Q_EMIT message(tr("Payment request file handling"), - tr("Cannot process payment request because BIP70 support was not compiled in.")+ + tr("Cannot process payment request because BIP70 is not supported.")+ tr("Due to widespread security flaws in BIP70 it's strongly recommended that any merchant instructions to switch wallets be ignored.")+ tr("If you are receiving this error you should request the merchant provide a BIP21 compatible URI."), CClientUIInterface::ICON_WARNING); -#endif } } @@ -400,440 +290,3 @@ void PaymentServer::setOptionsModel(OptionsModel *_optionsModel) { this->optionsModel = _optionsModel; } - -#ifdef ENABLE_BIP70 -struct X509StoreDeleter { - void operator()(X509_STORE* b) { - X509_STORE_free(b); - } -}; - -struct X509Deleter { - void operator()(X509* b) { X509_free(b); } -}; - -namespace // Anon namespace -{ - std::unique_ptr<X509_STORE, X509StoreDeleter> certStore; -} - -static void ReportInvalidCertificate(const QSslCertificate& cert) -{ - qDebug() << QString("%1: Payment server found an invalid certificate: ").arg(__func__) << cert.serialNumber() << cert.subjectInfo(QSslCertificate::CommonName) << cert.subjectInfo(QSslCertificate::DistinguishedNameQualifier) << cert.subjectInfo(QSslCertificate::OrganizationalUnitName); -} - -// -// Load OpenSSL's list of root certificate authorities -// -void PaymentServer::LoadRootCAs(X509_STORE* _store) -{ - // Unit tests mostly use this, to pass in fake root CAs: - if (_store) - { - certStore.reset(_store); - return; - } - - // Normal execution, use either -rootcertificates or system certs: - certStore.reset(X509_STORE_new()); - - // Note: use "-system-" default here so that users can pass -rootcertificates="" - // and get 'I don't like X.509 certificates, don't trust anybody' behavior: - QString certFile = QString::fromStdString(gArgs.GetArg("-rootcertificates", "-system-")); - - // Empty store - if (certFile.isEmpty()) { - qDebug() << QString("PaymentServer::%1: Payment request authentication via X.509 certificates disabled.").arg(__func__); - return; - } - - QList<QSslCertificate> certList; - - if (certFile != "-system-") { - qDebug() << QString("PaymentServer::%1: Using \"%2\" as trusted root certificate.").arg(__func__).arg(certFile); - - certList = QSslCertificate::fromPath(certFile); - // Use those certificates when fetching payment requests, too: - QSslConfiguration::defaultConfiguration().setCaCertificates(certList); - } else - certList = QSslConfiguration::systemCaCertificates(); - - int nRootCerts = 0; - const QDateTime currentTime = QDateTime::currentDateTime(); - - for (const QSslCertificate& cert : certList) { - // Don't log NULL certificates - if (cert.isNull()) - continue; - - // Not yet active/valid, or expired certificate - if (currentTime < cert.effectiveDate() || currentTime > cert.expiryDate()) { - ReportInvalidCertificate(cert); - continue; - } - - // Blacklisted certificate - if (cert.isBlacklisted()) { - ReportInvalidCertificate(cert); - continue; - } - - QByteArray certData = cert.toDer(); - const unsigned char *data = (const unsigned char *)certData.data(); - - std::unique_ptr<X509, X509Deleter> x509(d2i_X509(0, &data, certData.size())); - if (x509 && X509_STORE_add_cert(certStore.get(), x509.get())) - { - // Note: X509_STORE increases the reference count to the X509 object, - // we still have to release our reference to it. - ++nRootCerts; - } - else - { - ReportInvalidCertificate(cert); - continue; - } - } - qInfo() << "PaymentServer::LoadRootCAs: Loaded " << nRootCerts << " root certificates"; - - // Project for another day: - // Fetch certificate revocation lists, and add them to certStore. - // Issues to consider: - // performance (start a thread to fetch in background?) - // privacy (fetch through tor/proxy so IP address isn't revealed) - // would it be easier to just use a compiled-in blacklist? - // or use Qt's blacklist? - // "certificate stapling" with server-side caching is more efficient -} - -void PaymentServer::initNetManager() -{ - if (!optionsModel) - return; - delete netManager; - - // netManager is used to fetch paymentrequests given in bitcoin: URIs - netManager = new QNetworkAccessManager(this); - - QNetworkProxy proxy; - - // Query active SOCKS5 proxy - if (optionsModel->getProxySettings(proxy)) { - netManager->setProxy(proxy); - - qDebug() << "PaymentServer::initNetManager: Using SOCKS5 proxy" << proxy.hostName() << ":" << proxy.port(); - } - else - qDebug() << "PaymentServer::initNetManager: No active proxy server found."; - - connect(netManager, &QNetworkAccessManager::finished, this, &PaymentServer::netRequestFinished); - connect(netManager, &QNetworkAccessManager::sslErrors, this, &PaymentServer::reportSslErrors); -} - -// -// Warning: readPaymentRequestFromFile() is used in ipcSendCommandLine() -// so don't use "Q_EMIT message()", but "QMessageBox::"! -// -bool PaymentServer::readPaymentRequestFromFile(const QString& filename, PaymentRequestPlus& request) -{ - QFile f(filename); - if (!f.open(QIODevice::ReadOnly)) { - qWarning() << QString("PaymentServer::%1: Failed to open %2").arg(__func__).arg(filename); - return false; - } - - // BIP70 DoS protection - if (!verifySize(f.size())) { - return false; - } - - QByteArray data = f.readAll(); - - return request.parse(data); -} - -bool PaymentServer::processPaymentRequest(const PaymentRequestPlus& request, SendCoinsRecipient& recipient) -{ - if (!optionsModel) - return false; - - if (request.IsInitialized()) { - // Payment request network matches client network? - if (!verifyNetwork(optionsModel->node(), request.getDetails())) { - Q_EMIT message(tr("Payment request rejected"), tr("Payment request network doesn't match client network."), - CClientUIInterface::MSG_ERROR); - - return false; - } - - // Make sure any payment requests involved are still valid. - // This is re-checked just before sending coins in WalletModel::sendCoins(). - if (verifyExpired(request.getDetails())) { - Q_EMIT message(tr("Payment request rejected"), tr("Payment request expired."), - CClientUIInterface::MSG_ERROR); - - return false; - } - } else { - Q_EMIT message(tr("Payment request error"), tr("Payment request is not initialized."), - CClientUIInterface::MSG_ERROR); - - return false; - } - - recipient.paymentRequest = request; - recipient.message = GUIUtil::HtmlEscape(request.getDetails().memo()); - - request.getMerchant(certStore.get(), recipient.authenticatedMerchant); - - QList<std::pair<CScript, CAmount> > sendingTos = request.getPayTo(); - QStringList addresses; - - for (const std::pair<CScript, CAmount>& sendingTo : sendingTos) { - // Extract and check destination addresses - CTxDestination dest; - if (ExtractDestination(sendingTo.first, dest)) { - // Append destination address - addresses.append(QString::fromStdString(EncodeDestination(dest))); - } - else if (!recipient.authenticatedMerchant.isEmpty()) { - // Unauthenticated payment requests to custom bitcoin addresses are not supported - // (there is no good way to tell the user where they are paying in a way they'd - // have a chance of understanding). - Q_EMIT message(tr("Payment request rejected"), - tr("Unverified payment requests to custom payment scripts are unsupported."), - CClientUIInterface::MSG_ERROR); - return false; - } - - // Bitcoin amounts are stored as (optional) uint64 in the protobuf messages (see paymentrequest.proto), - // but CAmount is defined as int64_t. Because of that we need to verify that amounts are in a valid range - // and no overflow has happened. - if (!verifyAmount(sendingTo.second)) { - Q_EMIT message(tr("Payment request rejected"), tr("Invalid payment request."), CClientUIInterface::MSG_ERROR); - return false; - } - - // Extract and check amounts - CTxOut txOut(sendingTo.second, sendingTo.first); - if (IsDust(txOut, optionsModel->node().getDustRelayFee())) { - Q_EMIT message(tr("Payment request error"), tr("Requested payment amount of %1 is too small (considered dust).") - .arg(BitcoinUnits::formatWithUnit(optionsModel->getDisplayUnit(), sendingTo.second)), - CClientUIInterface::MSG_ERROR); - - return false; - } - - recipient.amount += sendingTo.second; - // Also verify that the final amount is still in a valid range after adding additional amounts. - if (!verifyAmount(recipient.amount)) { - Q_EMIT message(tr("Payment request rejected"), tr("Invalid payment request."), CClientUIInterface::MSG_ERROR); - return false; - } - } - // Store addresses and format them to fit nicely into the GUI - recipient.address = addresses.join("<br />"); - - if (!recipient.authenticatedMerchant.isEmpty()) { - qDebug() << "PaymentServer::processPaymentRequest: Secure payment request from " << recipient.authenticatedMerchant; - } - else { - qDebug() << "PaymentServer::processPaymentRequest: Insecure payment request to " << addresses.join(", "); - } - - return true; -} - -void PaymentServer::fetchRequest(const QUrl& url) -{ - QNetworkRequest netRequest; - netRequest.setAttribute(QNetworkRequest::User, BIP70_MESSAGE_PAYMENTREQUEST); - netRequest.setUrl(url); - netRequest.setRawHeader("User-Agent", CLIENT_NAME.c_str()); - netRequest.setRawHeader("Accept", BIP71_MIMETYPE_PAYMENTREQUEST); - netManager->get(netRequest); -} - -void PaymentServer::fetchPaymentACK(WalletModel* walletModel, const SendCoinsRecipient& recipient, QByteArray transaction) -{ - const payments::PaymentDetails& details = recipient.paymentRequest.getDetails(); - if (!details.has_payment_url()) - return; - - QNetworkRequest netRequest; - netRequest.setAttribute(QNetworkRequest::User, BIP70_MESSAGE_PAYMENTACK); - netRequest.setUrl(QString::fromStdString(details.payment_url())); - netRequest.setHeader(QNetworkRequest::ContentTypeHeader, BIP71_MIMETYPE_PAYMENT); - netRequest.setRawHeader("User-Agent", CLIENT_NAME.c_str()); - netRequest.setRawHeader("Accept", BIP71_MIMETYPE_PAYMENTACK); - - payments::Payment payment; - payment.set_merchant_data(details.merchant_data()); - payment.add_transactions(transaction.data(), transaction.size()); - - // Create a new refund address, or re-use: - CTxDestination dest; - const OutputType change_type = walletModel->wallet().getDefaultChangeType() != OutputType::CHANGE_AUTO ? walletModel->wallet().getDefaultChangeType() : walletModel->wallet().getDefaultAddressType(); - if (walletModel->wallet().getNewDestination(change_type, "", dest)) { - // BIP70 requests encode the scriptPubKey directly, so we are not restricted to address - // types supported by the receiver. As a result, we choose the address format we also - // use for change. Despite an actual payment and not change, this is a close match: - // it's the output type we use subject to privacy issues, but not restricted by what - // other software supports. - std::string label = tr("Refund from %1").arg(recipient.authenticatedMerchant).toStdString(); - walletModel->wallet().setAddressBook(dest, label, "refund"); - - CScript s = GetScriptForDestination(dest); - payments::Output* refund_to = payment.add_refund_to(); - refund_to->set_script(&s[0], s.size()); - } else { - // This should never happen, because sending coins should have - // just unlocked the wallet and refilled the keypool. - qWarning() << "PaymentServer::fetchPaymentACK: Error getting refund key, refund_to not set"; - } - - int length = payment.ByteSize(); - netRequest.setHeader(QNetworkRequest::ContentLengthHeader, length); - QByteArray serData(length, '\0'); - if (payment.SerializeToArray(serData.data(), length)) { - netManager->post(netRequest, serData); - } - else { - // This should never happen, either. - qWarning() << "PaymentServer::fetchPaymentACK: Error serializing payment message"; - } -} - -void PaymentServer::netRequestFinished(QNetworkReply* reply) -{ - reply->deleteLater(); - - // BIP70 DoS protection - if (!verifySize(reply->size())) { - Q_EMIT message(tr("Payment request rejected"), - tr("Payment request %1 is too large (%2 bytes, allowed %3 bytes).") - .arg(reply->request().url().toString()) - .arg(reply->size()) - .arg(BIP70_MAX_PAYMENTREQUEST_SIZE), - CClientUIInterface::MSG_ERROR); - return; - } - - if (reply->error() != QNetworkReply::NoError) { - QString msg = tr("Error communicating with %1: %2") - .arg(reply->request().url().toString()) - .arg(reply->errorString()); - - qWarning() << "PaymentServer::netRequestFinished: " << msg; - Q_EMIT message(tr("Payment request error"), msg, CClientUIInterface::MSG_ERROR); - return; - } - - QByteArray data = reply->readAll(); - - QString requestType = reply->request().attribute(QNetworkRequest::User).toString(); - if (requestType == BIP70_MESSAGE_PAYMENTREQUEST) - { - PaymentRequestPlus request; - SendCoinsRecipient recipient; - if (!request.parse(data)) - { - qWarning() << "PaymentServer::netRequestFinished: Error parsing payment request"; - Q_EMIT message(tr("Payment request error"), - tr("Payment request cannot be parsed!"), - CClientUIInterface::MSG_ERROR); - } - else if (processPaymentRequest(request, recipient)) - Q_EMIT receivedPaymentRequest(recipient); - - return; - } - else if (requestType == BIP70_MESSAGE_PAYMENTACK) - { - payments::PaymentACK paymentACK; - if (!paymentACK.ParseFromArray(data.data(), data.size())) - { - QString msg = tr("Bad response from server %1") - .arg(reply->request().url().toString()); - - qWarning() << "PaymentServer::netRequestFinished: " << msg; - Q_EMIT message(tr("Payment request error"), msg, CClientUIInterface::MSG_ERROR); - } - else - { - Q_EMIT receivedPaymentACK(GUIUtil::HtmlEscape(paymentACK.memo())); - } - } -} - -void PaymentServer::reportSslErrors(QNetworkReply* reply, const QList<QSslError> &errs) -{ - Q_UNUSED(reply); - - QString errString; - for (const QSslError& err : errs) { - qWarning() << "PaymentServer::reportSslErrors: " << err; - errString += err.errorString() + "\n"; - } - Q_EMIT message(tr("Network request error"), errString, CClientUIInterface::MSG_ERROR); -} - -void PaymentServer::handlePaymentACK(const QString& paymentACKMsg) -{ - // currently we don't further process or store the paymentACK message - Q_EMIT message(tr("Payment acknowledged"), paymentACKMsg, CClientUIInterface::ICON_INFORMATION | CClientUIInterface::MODAL); -} - -bool PaymentServer::verifyNetwork(interfaces::Node& node, const payments::PaymentDetails& requestDetails) -{ - bool fVerified = requestDetails.network() == node.getNetwork(); - if (!fVerified) { - qWarning() << QString("PaymentServer::%1: Payment request network \"%2\" doesn't match client network \"%3\".") - .arg(__func__) - .arg(QString::fromStdString(requestDetails.network())) - .arg(QString::fromStdString(node.getNetwork())); - } - return fVerified; -} - -bool PaymentServer::verifyExpired(const payments::PaymentDetails& requestDetails) -{ - bool fVerified = (requestDetails.has_expires() && (int64_t)requestDetails.expires() < GetTime()); - if (fVerified) { - const QString requestExpires = QString::fromStdString(FormatISO8601DateTime((int64_t)requestDetails.expires())); - qWarning() << QString("PaymentServer::%1: Payment request expired \"%2\".") - .arg(__func__) - .arg(requestExpires); - } - return fVerified; -} - -bool PaymentServer::verifySize(qint64 requestSize) -{ - bool fVerified = (requestSize <= BIP70_MAX_PAYMENTREQUEST_SIZE); - if (!fVerified) { - qWarning() << QString("PaymentServer::%1: Payment request too large (%2 bytes, allowed %3 bytes).") - .arg(__func__) - .arg(requestSize) - .arg(BIP70_MAX_PAYMENTREQUEST_SIZE); - } - return fVerified; -} - -bool PaymentServer::verifyAmount(const CAmount& requestAmount) -{ - bool fVerified = MoneyRange(requestAmount); - if (!fVerified) { - qWarning() << QString("PaymentServer::%1: Payment request amount out of allowed range (%2, allowed 0 - %3).") - .arg(__func__) - .arg(requestAmount) - .arg(MAX_MONEY); - } - return fVerified; -} - -X509_STORE* PaymentServer::getCertStore() -{ - return certStore.get(); -} -#endif diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 30b5bc3b6d..8b2533508d 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -36,9 +36,6 @@ #include <config/bitcoin-config.h> #endif -#ifdef ENABLE_BIP70 -#include <qt/paymentrequestplus.h> -#endif #include <qt/walletmodel.h> #include <QObject> @@ -50,15 +47,9 @@ QT_BEGIN_NAMESPACE class QApplication; class QByteArray; class QLocalServer; -class QNetworkAccessManager; -class QNetworkReply; -class QSslError; class QUrl; QT_END_NAMESPACE -// BIP70 max payment request size in bytes (DoS protection) -static const qint64 BIP70_MAX_PAYMENTREQUEST_SIZE = 50000; - class PaymentServer : public QObject { Q_OBJECT @@ -82,27 +73,6 @@ public: // OptionsModel is used for getting proxy settings and display unit void setOptionsModel(OptionsModel *optionsModel); -#ifdef ENABLE_BIP70 - // Load root certificate authorities. Pass nullptr (default) - // to read from the file specified in the -rootcertificates setting, - // or, if that's not set, to use the system default root certificates. - // If you pass in a store, you should not X509_STORE_free it: it will be - // freed either at exit or when another set of CAs are loaded. - static void LoadRootCAs(X509_STORE* store = nullptr); - - // Return certificate store - static X509_STORE* getCertStore(); - - // Verify that the payment request network matches the client network - static bool verifyNetwork(interfaces::Node& node, const payments::PaymentDetails& requestDetails); - // Verify if the payment request is expired - static bool verifyExpired(const payments::PaymentDetails& requestDetails); - // Verify the payment request size is valid as per BIP70 - static bool verifySize(qint64 requestSize); - // Verify the payment request amount is valid - static bool verifyAmount(const CAmount& requestAmount); -#endif - Q_SIGNALS: // Fired when a valid payment request is received void receivedPaymentRequest(SendCoinsRecipient); @@ -110,11 +80,6 @@ Q_SIGNALS: // Fired when a message should be reported to the user void message(const QString &title, const QString &message, unsigned int style); -#ifdef ENABLE_BIP70 - // Fired when a valid PaymentACK is received - void receivedPaymentACK(const QString &paymentACKMsg); -#endif - public Q_SLOTS: // Signal this when the main window's UI is ready // to display payment requests to the user @@ -123,18 +88,8 @@ public Q_SLOTS: // Handle an incoming URI, URI with local file scheme or file void handleURIOrFile(const QString& s); -#ifdef ENABLE_BIP70 - // Submit Payment message to a merchant, get back PaymentACK: - void fetchPaymentACK(WalletModel* walletModel, const SendCoinsRecipient& recipient, QByteArray transaction); -#endif - private Q_SLOTS: void handleURIConnection(); -#ifdef ENABLE_BIP70 - void netRequestFinished(QNetworkReply*); - void reportSslErrors(QNetworkReply*, const QList<QSslError> &); - void handlePaymentACK(const QString& paymentACKMsg); -#endif protected: // Constructor registers this on the parent QApplication to @@ -145,16 +100,6 @@ private: bool saveURIs; // true during startup QLocalServer* uriServer; OptionsModel *optionsModel; - -#ifdef ENABLE_BIP70 - static bool readPaymentRequestFromFile(const QString& filename, PaymentRequestPlus& request); - bool processPaymentRequest(const PaymentRequestPlus& request, SendCoinsRecipient& recipient); - void fetchRequest(const QUrl& url); - - // Setup networking - void initNetManager(); - QNetworkAccessManager* netManager; // Used to fetch payment requests -#endif }; #endif // BITCOIN_QT_PAYMENTSERVER_H diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index bf1baf5470..2332d52b9a 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -71,6 +71,7 @@ bool QRImageWidget::setQR(const QString& data, const QString& text) if (!text.isEmpty()) { QFont font = GUIUtil::fixedPitchFont(); + font.setStyleStrategy(QFont::NoAntialias); QRect paddedRect = qrAddrImage.rect(); // calculate ideal font size diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 003a31b248..8edcca684d 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -291,9 +291,6 @@ void SendCoinsDialog::on_sendButton_clicked() QString recipientElement; -#ifdef ENABLE_BIP70 - if (!rcp.paymentRequest.IsInitialized()) // normal payment -#endif { if(rcp.label.length() > 0) // label with address { @@ -305,17 +302,6 @@ void SendCoinsDialog::on_sendButton_clicked() recipientElement.append(tr("%1 to %2").arg(amount, address)); } } -#ifdef ENABLE_BIP70 - else if(!rcp.authenticatedMerchant.isEmpty()) // authenticated payment request - { - recipientElement.append(tr("%1 to '%2'").arg(amount, rcp.authenticatedMerchant)); - } - else // unauthenticated payment request - { - recipientElement.append(tr("%1 to %2").arg(amount, address)); - } -#endif - formatted.append(recipientElement); } @@ -558,8 +544,7 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.second = CClientUIInterface::MSG_WARNING; // This comment is specific to SendCoinsDialog usage of WalletModel::SendCoinsReturn. - // WalletModel::TransactionCommitFailed is used only in WalletModel::sendCoins() - // all others are used only in WalletModel::prepareTransaction() + // All status values are used only in WalletModel::prepareTransaction() switch(sendCoinsReturn.status) { case WalletModel::InvalidAddress: @@ -581,10 +566,6 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.first = tr("Transaction creation failed!"); msgParams.second = CClientUIInterface::MSG_ERROR; break; - case WalletModel::TransactionCommitFailed: - msgParams.first = tr("The transaction was rejected with the following reason: %1").arg(sendCoinsReturn.reasonCommitFailed); - msgParams.second = CClientUIInterface::MSG_ERROR; - break; case WalletModel::AbsurdFee: msgParams.first = tr("A fee higher than %1 is considered an absurdly high fee.").arg(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), model->wallet().getDefaultMaxTxFee())); break; diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index 7324d759fb..be417655b4 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -137,12 +137,6 @@ bool SendCoinsEntry::validate(interfaces::Node& node) // Check input validity bool retval = true; -#ifdef ENABLE_BIP70 - // Skip checks for payment request - if (recipient.paymentRequest.IsInitialized()) - return retval; -#endif - if (!model->validateAddress(ui->payTo->text())) { ui->payTo->setValid(false); @@ -172,13 +166,6 @@ bool SendCoinsEntry::validate(interfaces::Node& node) SendCoinsRecipient SendCoinsEntry::getValue() { -#ifdef ENABLE_BIP70 - // Payment request - if (recipient.paymentRequest.IsInitialized()) - return recipient; -#endif - - // Normal payment recipient.address = ui->payTo->text(); recipient.label = ui->addAsLabel->text(); recipient.amount = ui->payAmount->value(); @@ -203,29 +190,6 @@ QWidget *SendCoinsEntry::setupTabChain(QWidget *prev) void SendCoinsEntry::setValue(const SendCoinsRecipient &value) { recipient = value; - -#ifdef ENABLE_BIP70 - if (recipient.paymentRequest.IsInitialized()) // payment request - { - if (recipient.authenticatedMerchant.isEmpty()) // unauthenticated - { - ui->payTo_is->setText(recipient.address); - ui->memoTextLabel_is->setText(recipient.message); - ui->payAmount_is->setValue(recipient.amount); - ui->payAmount_is->setReadOnly(true); - setCurrentWidget(ui->SendCoins_UnauthenticatedPaymentRequest); - } - else // authenticated - { - ui->payTo_s->setText(recipient.authenticatedMerchant); - ui->memoTextLabel_s->setText(recipient.message); - ui->payAmount_s->setValue(recipient.amount); - ui->payAmount_s->setReadOnly(true); - setCurrentWidget(ui->SendCoins_AuthenticatedPaymentRequest); - } - } - else // normal payment -#endif { // message ui->messageTextLabel->setText(recipient.message); diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 4fe440a679..6e8d383847 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,6 +1,6 @@ #include <qt/test/addressbooktests.h> #include <qt/test/util.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <interfaces/chain.h> #include <interfaces/node.h> @@ -51,11 +51,10 @@ void EditAddressAndSubmit( * In each case, verify the resulting state of the address book and optionally * the warning message presented to the user. */ -void TestAddAddressesToSendBook() +void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; - auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); @@ -101,10 +100,9 @@ void TestAddAddressesToSendBook() // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); - auto node = interfaces::MakeNode(); - OptionsModel optionsModel(*node); + OptionsModel optionsModel(node); AddWallet(wallet); - WalletModel walletModel(std::move(node->getWallets()[0]), *node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); RemoveWallet(wallet); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); @@ -150,5 +148,5 @@ void AddressBookTests::addressBookTests() return; } #endif - TestAddAddressesToSendBook(); + TestAddAddressesToSendBook(m_node); } diff --git a/src/qt/test/addressbooktests.h b/src/qt/test/addressbooktests.h index beeb9e76a9..9944750ec8 100644 --- a/src/qt/test/addressbooktests.h +++ b/src/qt/test/addressbooktests.h @@ -4,8 +4,16 @@ #include <QObject> #include <QTest> +namespace interfaces { +class Node; +} // namespace interfaces + class AddressBookTests : public QObject { +public: + AddressBookTests(interfaces::Node& node) : m_node(node) {} + interfaces::Node& m_node; + Q_OBJECT private Q_SLOTS: diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index e730c8f6d5..664826ecf2 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -11,7 +11,7 @@ #include <qt/networkstyle.h> #include <qt/rpcconsole.h> #include <shutdown.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <univalue.h> #include <validation.h> diff --git a/src/qt/test/compattests.cpp b/src/qt/test/compattests.cpp index 6750c543da..cf86a5bc1e 100644 --- a/src/qt/test/compattests.cpp +++ b/src/qt/test/compattests.cpp @@ -6,10 +6,6 @@ #include <config/bitcoin-config.h> #endif -#if defined(ENABLE_WALLET) && defined(ENABLE_BIP70) -#include <qt/paymentrequestplus.h> // this includes protobuf's port.h which defines its own bswap macos -#endif - #include <qt/test/compattests.h> #include <compat/byteswap.h> diff --git a/src/qt/test/paymentrequestdata.h b/src/qt/test/paymentrequestdata.h deleted file mode 100644 index 7f45d30973..0000000000 --- a/src/qt/test/paymentrequestdata.h +++ /dev/null @@ -1,465 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_QT_TEST_PAYMENTREQUESTDATA_H -#define BITCOIN_QT_TEST_PAYMENTREQUESTDATA_H - -// -// Data for paymentservertests.cpp -// - -// Base64/DER-encoded fake certificate authority certificates. -// Convert pem to base64/der with: -// openssl x509 -in cert.pem -inform PEM -outform DER | openssl enc -base64 - -// Serial Number: 10302349811211485352 (0x8ef94c91b112c0a8) -// Issuer: CN=PaymentRequest Test CA -// Subject: CN=PaymentRequest Test CA -// Not Valid After : Dec 8 16:37:24 2022 GMT -// -const char* caCert1_BASE64 = -"\ -MIIB0DCCATmgAwIBAgIJAI75TJGxEsCoMA0GCSqGSIb3DQEBCwUAMCExHzAdBgNV\ -BAMTFlBheW1lbnRSZXF1ZXN0IFRlc3QgQ0EwHhcNMTIxMjEwMTYzNzI0WhcNMjIx\ -MjA4MTYzNzI0WjAhMR8wHQYDVQQDExZQYXltZW50UmVxdWVzdCBUZXN0IENBMIGf\ -MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCvua59nX9radoqDYyplcns5qdVDTN1\ -7tmcGixmMYOYU3UYMU55VSsJs0dWKnMm3COQDY+N63c0XSbRqarBcsLTkaNASuPX\ -FCv1VWuEKSyy5xe4zeoDU7CVSzlxtQD9wbZW/s3ISjgaXBpwn6eVmntb0JwYxxPc\ -M1u/hrMD8BDbSQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA\ -A4GBADSaRgK5xe47XxycXBhHhr0Wgl4pAsFsufqA9aB9r8KNEHJ0yUvvbD/jaJJM\ -RtQcf0AJ9olzUMY4syehxbzUJP6aeXhZEYiMvdvcv9D55clq6+WLLlNT3jBgAaVn\ -p3waRjPD4bUX3nv+ojz5s4puw7Qq5QUZlhGsMzPvwDGCmZkL\ -"; - -// Serial Number: f0:da:97:e4:38:d7:64:16 -// Issuer: CN=PaymentRequest Test CA -// Subject: CN=PaymentRequest Test CA -// Not Valid After : Jan 8 18:21:06 2025 GMT -// -const char* caCert2_BASE64 = -"\ -MIIC1TCCAb2gAwIBAgIJAPDal+Q412QWMA0GCSqGSIb3DQEBCwUAMCExHzAdBgNV\ -BAMMFlBheW1lbnRSZXF1ZXN0IFRlc3QgQ0EwHhcNMTUwMTExMTgyMTA2WhcNMjUw\ -MTA4MTgyMTA2WjAhMR8wHQYDVQQDDBZQYXltZW50UmVxdWVzdCBUZXN0IENBMIIB\ -IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1S9wVLfTplJuT/1OaaBgl/Mb\ -I392v8S9kHbzYz7B4OTMslaO7piz0v3SO3TKMh0dswjiRdHrIgpO7XdIUQiU/ugg\ -xDw0kuNehfz1ycaGedlFFtFHTNXqLyIUF3dlwHhQwaomM6RXoJmxLny5BhYHEcmk\ -yWwr3Cdjd9gAZpblugVJB9C1e40uyL8ao4PHdLzOqO27iSe6riP8SwwisJZEbMaz\ -AZpgNEEMbIXPJEFvm5HTRXSMtQCOTSZYMFF0M2yrtmlECnz7hWP19b9bcoDzZQB4\ -ylIsFG/7q2jV7MC/e2STZv+niJiHL08RUdoFpAgzaxMgqj63C7B55HgNDNHJYQID\ -AQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBGejPxLxj9\ -+crv6gUeEBMZPiUx7pUgcI22Wm5yymP96B4fwI3Y0DBehq20d76vbWGPN17Z6pH3\ -ge7PVY1SYqXtS6hXTo4olCm/BZADli+2Bs2xCiaa+Ltve4ufVej+bKJXN/YnrhvO\ -Kq+klQkuuHywU+GJV/NQeBqToIrSOBgi477NgLFCCCmmx2QWsxHoCFGfuRCBVseT\ -z2k/tMuALCDXGeZBRPTsGHu1y4cj84swAeoDK5QSQcI+Ub7GKc+zkoj02sdDLiMo\ -3wokYPcIy47oclhmb4xubHc+y7nF610yZBoC/zgbhbawnZ65hDDWkdQ/SVAnWZD7\ -9PFfmNnYPTQH\ -"; - -// -// This payment request validates directly against the -// caCert1 certificate authority. -// -const char* paymentrequest1_cert1_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrxAwruAzCCAeowggFToAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAxMWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xMjEyMTAx\ -NjM3MjRaFw0yMjEyMDgxNjM3MjRaMEMxGTAXBgNVBAMMEHRlc3RtZXJjaGFudC5v\ -cmcxJjAkBgNVBAoMHVBheW1lbnQgUmVxdWVzdCBUZXN0IE1lcmNoYW50MIGfMA0G\ -CSqGSIb3DQEBAQUAA4GNADCBiQKBgQDHkMy8W1u6HsWlSqdWTmMKf54gICxNfxbY\ -+rcMtAftr62hCYx2d2QiSRd1pCUzmo12IiSX3WxSHwaTnT3MFD6jRx6+zM6XdGar\ -I2zpYle11ANzu4gAthN17uRQHV2O5QxVtzNaMdKeJLXT2L9tfEdyL++9ZUqoQmdA\ -YG9ix330hQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB\ -AIkyO99KC68bi9PFRyQQ7nvn5GlQEb3Ca1bRG5+AKN9N5vc8rZ9G2hejtM8wEXni\ -eGBP+chVMsbTPEHKLrwREn7IvcyCcbAStaklPC3w0B/2idQSHskb6P3X13OR2bTH\ -a2+6wuhsOZRUrVNr24rM95DKx/eCC6JN1VW+qRPU6fqzIjQSHwiw2wYSGXapFJVg\ -igPI+6XpExtNLO/i1WFV8ZmoiKwYsuHFiwUqC1VuaXRUZXN0T25lKoABS0j59iMU\ -Uc9MdIfwsO1BskIET0eJSGNZ7eXb9N62u+qf831PMpEHkmlGpk8rHy92nPcgua/U\ -Yt8oZMn3QaTZ5A6HjJbc3A73eLylp1a0SwCl+KDMEvDQhqMn1jAVu2v92AH3uB7n\ -SiWVbw0tX/68iSQEGGfh9n6ee/8Myb3ICdw=\ -"; - -// -// Signed, but expired, merchant cert in the request -// -const char* paymentrequest2_cert1_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrsAwrpAzCCAeUwggFOoAMCAQICAQMwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAxMWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xMzAyMjMy\ -MTI2NDNaFw0xMzAyMjQyMTI2NDNaMD4xHDAaBgNVBAMME2V4cGlyZWRtZXJjaGFu\ -dC5vcmcxHjAcBgNVBAoMFUV4cGlyZWQgVGVzdCBNZXJjaGFudDCBnzANBgkqhkiG\ -9w0BAQEFAAOBjQAwgYkCgYEAx5DMvFtbuh7FpUqnVk5jCn+eICAsTX8W2Pq3DLQH\ -7a+toQmMdndkIkkXdaQlM5qNdiIkl91sUh8Gk509zBQ+o0cevszOl3RmqyNs6WJX\ -tdQDc7uIALYTde7kUB1djuUMVbczWjHSniS109i/bXxHci/vvWVKqEJnQGBvYsd9\ -9IUCAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQAaU137\ -j53rvSjlmYZpZ4RWTP7EdD6fl5ZxBeXHytN6DQL33H0eD7OFHt+ofc7E6D7keubl\ -UfCu+jOvt/MvvPUmtCI9yXZ0dNC4sjyETv+wQpxO0UNZwOM4uegdCzlo6Bi3pD4/\ -KKLdMkWuUfuPBmoammny74lZaOVr5deKXztTuCI0Eh8IsNsGEhl2qRSVYIoDyPul\ -6RMbTSzv4tVhVfGZqIisGLLhxYsFKgtVbml0VGVzdFR3byqAAXHuo4nZEPniLpkd\ -y30TkwBxVgprWJ18a9z/7Py35Qss/JMbOXbnBhJtmJCdIowHRI0aa+zqt3KKKAXi\ -mm+V4seMgxTcxMS+eDDkiTcB/RtWWSyRcS2ANjFeY0T4SLMwiCL9qWPi03hr8j96\ -tejrSPOBNSJ3Mi/q5u2Yl4gJZY2b\ -"; - -// -// 10-long certificate chain, all intermediates valid -// -const char* paymentrequest3_cert1_BASE64 = -"\ -Egt4NTA5K3NoYTI1Nhq8JAr/AzCCAfswggFkoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwPzEUMBIGA1UEAwwLdGVzdGNhOC5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVx\ -dWVzdCBJbnRlcm1lZGlhdGUgODAeFw0xMzAyMjMyMjQyMzFaFw0yMzAyMjEyMjQy\ -MzFaMDYxGjAYBgNVBAMMEXRlc3RtZXJjaGFudDgub3JnMRgwFgYDVQQKDA9UZXN0\ -IE1lcmNoYW50IDgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMMCHA3hiHbS\ -TKZ5K9jHRwE8NxkGp3IOx56PDB2diNkldG8XweTcRq7bBm7pdiBt4IVggtfs+6hE\ -hDYIOecyoAnVzPFTdvQ7KQdQ/fD9YLe6lk+o0edOqutPMyrxLFjSluXxEQyk7fdt\ -URloMMYfp3p1/hFCboA1rAsQ2RW38hR5AgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w\ -DQYJKoZIhvcNAQELBQADgYEAPsdFatnc2RJSpvZsw+nCiPVsllycw5ELglq9vfJz\ -nJJucRxgzmqI2iuas1ugwbXn0BEIRLK7vMF/qBzQR6M/nTxttah+KEu+okjps9vJ\ -cIyhfTyGPC5xkHaHZ7sG+UHOFhPw0/kXn0x+pbVgBZ5315axqcp1R+DTSj/whMAr\ -n0AKiAQwggIEMIIBbaADAgECAgECMA0GCSqGSIb3DQEBCwUAMD8xFDASBgNVBAMM\ -C3Rlc3RjYTcub3JnMScwJQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRp\ -YXRlIDcwHhcNMTMwMjIzMjI0MjMxWhcNMjMwMjIxMjI0MjMxWjA/MRQwEgYDVQQD\ -DAt0ZXN0Y2E4Lm9yZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVk\ -aWF0ZSA4MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDexUFfxb1sThvabp7u\ -dZz59ciThGmmAW0nP4tjrgEACgvWIInr2dZpTHbiQNF34ycsk0le1JD93D7Qb8rd\ -25OrpaO8XS2Li2zjR9cleixXjSLwV/zv8zJ8yPl/27XL++PDTKBXVpJ8/Syp+9Ty\ -plV1BqDhqtIHb/QSHEkTQXjeYQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqG\ -SIb3DQEBCwUAA4GBACMooQVbkbIZ2DaPwHDc4ULwguG3VI2Kzj50UdExmHtzm2S4\ -MQei+n+HEPjtJAx5OY520+10nfuP+12H2DRLQmWmdvDpeQ/Cv0yavlw4ZRejRFo7\ -KS83C0wo5rd+qTvvOmAN4UTArWkzYcEUulPdiXnRamb0WQHTeVdIbHVkMormCogE\ -MIICBDCCAW2gAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MRQwEgYDVQQDDAt0ZXN0\ -Y2E2Lm9yZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVkaWF0ZSA2\ -MB4XDTEzMDIyMzIyNDIzMVoXDTIzMDIyMTIyNDIzMVowPzEUMBIGA1UEAwwLdGVz\ -dGNhNy5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUg\ -NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtjBRazrkebXAhXsbjimrMIRm\ -W/f9SwAHwXfc042keNtl0t2z6XE6UPcR2v/KrssXuCZgodeYxz6IM6lWosCM1xot\ -C3ChKKFBfVO30reuKBRUxXfKAFqxaG0YOAEzdZkkY9AGhqWloeSmgxpIfhInU0EF\ -JjCwrJ6IkijBatGoAAECAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B\ -AQsFAAOBgQDBRTi1MolmOA0niHYX0A2lN5QWHkCfX0A7GwyoMA3dvM45m/NYd4WB\ -X+HwfnfYcI6X9jOgNo5OWmc4GGsld0HlxwMYEKISBS9PbSHPBrb3TBOlw5ztQpXZ\ -91+bOhLux52Fr03sK7v9qExmBM12M8UR2ltpzAMiUgLLMHyPfiWkvQqIBDCCAgQw\ -ggFtoAMCAQICAQIwDQYJKoZIhvcNAQELBQAwPzEUMBIGA1UEAwwLdGVzdGNhNS5v\ -cmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUgNTAeFw0x\ -MzAyMjMyMjQyMzBaFw0yMzAyMjEyMjQyMzBaMD8xFDASBgNVBAMMC3Rlc3RjYTYu\ -b3JnMScwJQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDYwgZ8w\ -DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANJSH3xivX1t9olIdHsznI1aE9SD7t9i\ -SZJsIB0otoETHZRVv9M9LvyzBNK98ZV+kTOlST7PJgC0d9BQM9sgYApSRq5oqKDM\ -9FXbOm/yaReAbU3mkFNFw5roTlJ5ThEy0yOGT/DS0YBRaGIvRPRj2DiqDVdCZZ+w\ -4jo1IYHkZt4FAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQAD\ -gYEATm6+J1OmbrothO60xALKonWMBKr6hudb4amkFBqKbA9wMeM3jl+I/yKfz/Uf\ -xWuJ071IhiNv6Gxx5YwNvhUe1xMhUqHv0gpyK1Z47bD+kYS2se5sWNPNo3Y9qZDG\ -IXiGQxwHmrzaFk79Uy1xsmvsEz42w6hr25Yaw7HkIgrFveoKiAQwggIEMIIBbaAD\ -AgECAgECMA0GCSqGSIb3DQEBCwUAMD8xFDASBgNVBAMMC3Rlc3RjYTQub3JnMScw\ -JQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDQwHhcNMTMwMjIz\ -MjI0MjMwWhcNMjMwMjIxMjI0MjMwWjA/MRQwEgYDVQQDDAt0ZXN0Y2E1Lm9yZzEn\ -MCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVkaWF0ZSA1MIGfMA0GCSqG\ -SIb3DQEBAQUAA4GNADCBiQKBgQC7vVUFpxHzz2Tr/xij3k58s8d/BPA0R6D5RXTV\ -vmhAzc1Zuin4zUKRFs/aCj/0yED8Wu/COfNGF4tVlRNMdl9EcFsxa8XGEL4eAZa+\ -H/rOHH+7/1EINrrVWhZlUecyhilN8jmCZmqEM3ecuD0NAViqyMrgmaiFmsLoQZpE\ -GepDUQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GBAEdJ\ -Ss8jWiooja3WZzHXeF95QkBJNjIlpDLGcpl4opOYLSuEl9Uxp//LaQQiXuzpj4/I\ -pkWGQmMy5HOyH1lqDyiMgXpcG8PE0jEQAoEUGZ0QEqB1mZ6BCrYvmUuf/5aSVd8Y\ -6lKMR3WzFDYU9Zy0nzuHB/3nvp6MeDRQeRMtYvz4CogEMIICBDCCAW2gAwIBAgIB\ -AjANBgkqhkiG9w0BAQsFADA/MRQwEgYDVQQDDAt0ZXN0Y2EzLm9yZzEnMCUGA1UE\ -CgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVkaWF0ZSAzMB4XDTEzMDIyMzIyNDIy\ -OVoXDTIzMDIyMTIyNDIyOVowPzEUMBIGA1UEAwwLdGVzdGNhNC5vcmcxJzAlBgNV\ -BAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUgNDCBnzANBgkqhkiG9w0B\ -AQEFAAOBjQAwgYkCgYEAxYYo3w2UXiYg6O8b4QgwN/vgreTkiW122Ep/z2TiDrhV\ -MhfOOiKdwYESPflfnXnVaQQzCGexYTQqsvqvzHSyna5hL0zPTRJxSKmTVrXRsWtp\ -dCRhjxCGipS3tlQBDi7vb+7SNRIBK4dBjjGzALNk7gMCpy+yM8f6I043jTlmGb0C\ -AwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQDU+IQxt3Oh\ -KqaUYWC23+cB2gekvWqwMBnrCNrX/Dp+kjoJKUoR2Fs3qw53raHES4SIhpGT9l9l\ -rppNQgFe/JMHeYqOZMZO+6kuU0olJanBJ14tPIc7zlMTQ9OfmZ6v07IpyFbsQDtR\ -hpe80DpuvSFPfJ4fh0WrQf6kn3KDVpGDnAqIBDCCAgQwggFtoAMCAQICAQIwDQYJ\ -KoZIhvcNAQELBQAwPzEUMBIGA1UEAwwLdGVzdGNhMi5vcmcxJzAlBgNVBAoMHlBh\ -eW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUgMjAeFw0xMzAyMjMyMjQyMjlaFw0y\ -MzAyMjEyMjQyMjlaMD8xFDASBgNVBAMMC3Rlc3RjYTMub3JnMScwJQYDVQQKDB5Q\ -YXltZW50IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDMwgZ8wDQYJKoZIhvcNAQEBBQAD\ -gY0AMIGJAoGBANzgVP99Qg98e6NsKEz1v5KqRB7NTBRRsYnBvb/TSWipvMQaCYuE\ -yk1xG57x++QuASKeR3QHRQJOoAhQaj9JLUhSSv9GQ5PrFLLsOFv7L1tpzXHh2dOB\ -IW92X2yFRW2s39q+Q21yvN+N8uoKdqXhzRA+dDoXh3cavaVeHX1G+IrlAgMBAAGj\ -EDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADgYEASTwg84cX+1UhOG9s\ -ejFV3m34QuI1hPZ+qhqVJlRYUtego8Wng1BburDSwqVAv4ch2wi3c2s4e8J7AXyL\ -tzSbSQG4RN0oZi0mR8EtTTN+Mix/hBIk79dMZg85+I29uFA6Zj2d9oAhQv2qkHhc\ -6tcaheNvkQRlCyH68k3iF1Fqf+4KiAQwggIEMIIBbaADAgECAgECMA0GCSqGSIb3\ -DQEBCwUAMD8xFDASBgNVBAMMC3Rlc3RjYTEub3JnMScwJQYDVQQKDB5QYXltZW50\ -IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDEwHhcNMTMwMjIzMjI0MjI5WhcNMjMwMjIx\ -MjI0MjI5WjA/MRQwEgYDVQQDDAt0ZXN0Y2EyLm9yZzEnMCUGA1UECgweUGF5bWVu\ -dCBSZXF1ZXN0IEludGVybWVkaWF0ZSAyMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB\ -iQKBgQDaV8zhfyQuSf/f+fauMfgs3g/RnWy9yxxUkvQneQQPH3uZzCyk3A6q72ip\ -TtwNqiibG9455L9A7SaUjGtnpUz0NKT/VWUdqbfCl1PqXjEZbDobbAQ5hxLGOTyL\ -RQhLIcgeq2/BnmeCqHsC4md04nUp+nBo1HwKyygvK+9sMbCp/wIDAQABoxAwDjAM\ -BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GBACvYyE+PPmWFkbjyRu9LAt8D\ -crtyYYLRClKSg6tVvutwukLG2l//kDOohYkJtgTqr6LnCIIIwYdXN+4wxugmw4cn\ -PIZmP6kovxjhhVM95okilor1zniTAo3RN7JDIfTGNgxLdGu1btt7DOFL4zTbeSJM\ -b8M1JpPftehH+x/VLyuUCuoDMIIB5jCCAU+gAwIBAgIBBTANBgkqhkiG9w0BAQsF\ -ADAhMR8wHQYDVQQDExZQYXltZW50UmVxdWVzdCBUZXN0IENBMB4XDTEzMDIyMzIy\ -NDIyOFoXDTIzMDIyMTIyNDIyOFowPzEUMBIGA1UEAwwLdGVzdGNhMS5vcmcxJzAl\ -BgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUgMTCBnzANBgkqhkiG\ -9w0BAQEFAAOBjQAwgYkCgYEAo5Vy9H3nA/OOkF5Ap89yfVNSiTay/LYCaB0eALpc\ -U690U75O9Q3w2M+2AN8wpbbHsJHZMIjEeBRoQfjlYXW1ucQTxWKyT+liu0D25mGX\ -X27CBXBd4iXTxVII/iX+u3lcjORjoHOBy7QgeIDIIS9y0vYu8eArpjh7m4thrVgI\ -RtMCAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQB9LKcV\ -JK9sjASNzpQlpUp7nCiw5FSjVY+XMRIKK/kavzlKjZ+InsmmyRVGjDoZi9GrqG9P\ -VHgLBxi2VtVjmokZoNPqao3OfhqORAubC+JR/JLepM7aDaxDdTHVhSUk4lgNAvi2\ -6dGY7nZMsnHlPQ2tPp/HvRRiMq1oDjlylc8VTCI2Eh8IsNsGEhl2qRSVYIoDyPul\ -6RMbTSzv4tVhVfGZqIisGLLhxYsFKg1Vbml0VGVzdFRocmVlKoABn2HTsUQtMNI4\ -yNvkfkFNka3pRvTUTydJrvyfmEeLzImfM1BWddZjnywku9RToNFZZNgow5QnljmF\ -chhR/aHOuEMTxmc12K4rNlgYtHCsxLP9zd+6u0cva3TucZ6EzS8PKEib/+r12/52\ -664NuWA9WtsK7QCFrK2K95PnVCRmWl0=\ -"; - -// -// Long certificate chain, with an expired certificate in the middle -// -const char* paymentrequest4_cert1_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhqeJAr/AzCCAfswggFkoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwPzEUMBIGA1UEAwwLdGVzdGNhOC5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVx\ -dWVzdCBJbnRlcm1lZGlhdGUgODAeFw0xMzAyMjMyMjQyMzFaFw0yMzAyMjEyMjQy\ -MzFaMDYxGjAYBgNVBAMMEXRlc3RtZXJjaGFudDgub3JnMRgwFgYDVQQKDA9UZXN0\ -IE1lcmNoYW50IDgwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMMCHA3hiHbS\ -TKZ5K9jHRwE8NxkGp3IOx56PDB2diNkldG8XweTcRq7bBm7pdiBt4IVggtfs+6hE\ -hDYIOecyoAnVzPFTdvQ7KQdQ/fD9YLe6lk+o0edOqutPMyrxLFjSluXxEQyk7fdt\ -URloMMYfp3p1/hFCboA1rAsQ2RW38hR5AgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w\ -DQYJKoZIhvcNAQELBQADgYEAPsdFatnc2RJSpvZsw+nCiPVsllycw5ELglq9vfJz\ -nJJucRxgzmqI2iuas1ugwbXn0BEIRLK7vMF/qBzQR6M/nTxttah+KEu+okjps9vJ\ -cIyhfTyGPC5xkHaHZ7sG+UHOFhPw0/kXn0x+pbVgBZ5315axqcp1R+DTSj/whMAr\ -n0AKiAQwggIEMIIBbaADAgECAgECMA0GCSqGSIb3DQEBCwUAMD8xFDASBgNVBAMM\ -C3Rlc3RjYTcub3JnMScwJQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRp\ -YXRlIDcwHhcNMTMwMjIzMjI0MjMxWhcNMjMwMjIxMjI0MjMxWjA/MRQwEgYDVQQD\ -DAt0ZXN0Y2E4Lm9yZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVk\ -aWF0ZSA4MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDexUFfxb1sThvabp7u\ -dZz59ciThGmmAW0nP4tjrgEACgvWIInr2dZpTHbiQNF34ycsk0le1JD93D7Qb8rd\ -25OrpaO8XS2Li2zjR9cleixXjSLwV/zv8zJ8yPl/27XL++PDTKBXVpJ8/Syp+9Ty\ -plV1BqDhqtIHb/QSHEkTQXjeYQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqG\ -SIb3DQEBCwUAA4GBACMooQVbkbIZ2DaPwHDc4ULwguG3VI2Kzj50UdExmHtzm2S4\ -MQei+n+HEPjtJAx5OY520+10nfuP+12H2DRLQmWmdvDpeQ/Cv0yavlw4ZRejRFo7\ -KS83C0wo5rd+qTvvOmAN4UTArWkzYcEUulPdiXnRamb0WQHTeVdIbHVkMormCogE\ -MIICBDCCAW2gAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MRQwEgYDVQQDDAt0ZXN0\ -Y2E2Lm9yZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVkaWF0ZSA2\ -MB4XDTEzMDIyMzIyNDIzMVoXDTIzMDIyMTIyNDIzMVowPzEUMBIGA1UEAwwLdGVz\ -dGNhNy5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUg\ -NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtjBRazrkebXAhXsbjimrMIRm\ -W/f9SwAHwXfc042keNtl0t2z6XE6UPcR2v/KrssXuCZgodeYxz6IM6lWosCM1xot\ -C3ChKKFBfVO30reuKBRUxXfKAFqxaG0YOAEzdZkkY9AGhqWloeSmgxpIfhInU0EF\ -JjCwrJ6IkijBatGoAAECAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B\ -AQsFAAOBgQDBRTi1MolmOA0niHYX0A2lN5QWHkCfX0A7GwyoMA3dvM45m/NYd4WB\ -X+HwfnfYcI6X9jOgNo5OWmc4GGsld0HlxwMYEKISBS9PbSHPBrb3TBOlw5ztQpXZ\ -91+bOhLux52Fr03sK7v9qExmBM12M8UR2ltpzAMiUgLLMHyPfiWkvQqIBDCCAgQw\ -ggFtoAMCAQICAQIwDQYJKoZIhvcNAQELBQAwPzEUMBIGA1UEAwwLdGVzdGNhNS5v\ -cmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUgNTAeFw0x\ -MzAyMjMyMjQyMzBaFw0yMzAyMjEyMjQyMzBaMD8xFDASBgNVBAMMC3Rlc3RjYTYu\ -b3JnMScwJQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDYwgZ8w\ -DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANJSH3xivX1t9olIdHsznI1aE9SD7t9i\ -SZJsIB0otoETHZRVv9M9LvyzBNK98ZV+kTOlST7PJgC0d9BQM9sgYApSRq5oqKDM\ -9FXbOm/yaReAbU3mkFNFw5roTlJ5ThEy0yOGT/DS0YBRaGIvRPRj2DiqDVdCZZ+w\ -4jo1IYHkZt4FAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQAD\ -gYEATm6+J1OmbrothO60xALKonWMBKr6hudb4amkFBqKbA9wMeM3jl+I/yKfz/Uf\ -xWuJ071IhiNv6Gxx5YwNvhUe1xMhUqHv0gpyK1Z47bD+kYS2se5sWNPNo3Y9qZDG\ -IXiGQxwHmrzaFk79Uy1xsmvsEz42w6hr25Yaw7HkIgrFveoK6gMwggHmMIIBT6AD\ -AgECAgEGMA0GCSqGSIb3DQEBCwUAMCExHzAdBgNVBAMTFlBheW1lbnRSZXF1ZXN0\ -IFRlc3QgQ0EwHhcNMTMwMjIzMjI1OTUxWhcNMTMwMjI0MjI1OTUxWjA/MRQwEgYD\ -VQQDDAt0ZXN0Y2E1Lm9yZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVy\ -bWVkaWF0ZSA1MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC7vVUFpxHzz2Tr\ -/xij3k58s8d/BPA0R6D5RXTVvmhAzc1Zuin4zUKRFs/aCj/0yED8Wu/COfNGF4tV\ -lRNMdl9EcFsxa8XGEL4eAZa+H/rOHH+7/1EINrrVWhZlUecyhilN8jmCZmqEM3ec\ -uD0NAViqyMrgmaiFmsLoQZpEGepDUQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0G\ -CSqGSIb3DQEBCwUAA4GBAEmcUEnhua/oiXy1fwScLgMqt+jk9mHRpE6SVsIop23Q\ -CY2JfpG6RxhMMzzzhGklEGN6cxG0HCi6B3HJx6PYrFEfTB0rW4K6m0Tvx3WpS9mN\ -uoEuJHLy18ausI/sYAPDHCL+SfBVcqorpaIG2sSpZouRBjRHAyqFAYlwlW87uq5n\ -CogEMIICBDCCAW2gAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MRQwEgYDVQQDDAt0\ -ZXN0Y2EzLm9yZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVkaWF0\ -ZSAzMB4XDTEzMDIyMzIyNDIyOVoXDTIzMDIyMTIyNDIyOVowPzEUMBIGA1UEAwwL\ -dGVzdGNhNC5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlh\ -dGUgNDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxYYo3w2UXiYg6O8b4Qgw\ -N/vgreTkiW122Ep/z2TiDrhVMhfOOiKdwYESPflfnXnVaQQzCGexYTQqsvqvzHSy\ -na5hL0zPTRJxSKmTVrXRsWtpdCRhjxCGipS3tlQBDi7vb+7SNRIBK4dBjjGzALNk\ -7gMCpy+yM8f6I043jTlmGb0CAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkqhkiG\ -9w0BAQsFAAOBgQDU+IQxt3OhKqaUYWC23+cB2gekvWqwMBnrCNrX/Dp+kjoJKUoR\ -2Fs3qw53raHES4SIhpGT9l9lrppNQgFe/JMHeYqOZMZO+6kuU0olJanBJ14tPIc7\ -zlMTQ9OfmZ6v07IpyFbsQDtRhpe80DpuvSFPfJ4fh0WrQf6kn3KDVpGDnAqIBDCC\ -AgQwggFtoAMCAQICAQIwDQYJKoZIhvcNAQELBQAwPzEUMBIGA1UEAwwLdGVzdGNh\ -Mi5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1lZGlhdGUgMjAe\ -Fw0xMzAyMjMyMjQyMjlaFw0yMzAyMjEyMjQyMjlaMD8xFDASBgNVBAMMC3Rlc3Rj\ -YTMub3JnMScwJQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDMw\ -gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANzgVP99Qg98e6NsKEz1v5KqRB7N\ -TBRRsYnBvb/TSWipvMQaCYuEyk1xG57x++QuASKeR3QHRQJOoAhQaj9JLUhSSv9G\ -Q5PrFLLsOFv7L1tpzXHh2dOBIW92X2yFRW2s39q+Q21yvN+N8uoKdqXhzRA+dDoX\ -h3cavaVeHX1G+IrlAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL\ -BQADgYEASTwg84cX+1UhOG9sejFV3m34QuI1hPZ+qhqVJlRYUtego8Wng1BburDS\ -wqVAv4ch2wi3c2s4e8J7AXyLtzSbSQG4RN0oZi0mR8EtTTN+Mix/hBIk79dMZg85\ -+I29uFA6Zj2d9oAhQv2qkHhc6tcaheNvkQRlCyH68k3iF1Fqf+4KiAQwggIEMIIB\ -baADAgECAgECMA0GCSqGSIb3DQEBCwUAMD8xFDASBgNVBAMMC3Rlc3RjYTEub3Jn\ -MScwJQYDVQQKDB5QYXltZW50IFJlcXVlc3QgSW50ZXJtZWRpYXRlIDEwHhcNMTMw\ -MjIzMjI0MjI5WhcNMjMwMjIxMjI0MjI5WjA/MRQwEgYDVQQDDAt0ZXN0Y2EyLm9y\ -ZzEnMCUGA1UECgweUGF5bWVudCBSZXF1ZXN0IEludGVybWVkaWF0ZSAyMIGfMA0G\ -CSqGSIb3DQEBAQUAA4GNADCBiQKBgQDaV8zhfyQuSf/f+fauMfgs3g/RnWy9yxxU\ -kvQneQQPH3uZzCyk3A6q72ipTtwNqiibG9455L9A7SaUjGtnpUz0NKT/VWUdqbfC\ -l1PqXjEZbDobbAQ5hxLGOTyLRQhLIcgeq2/BnmeCqHsC4md04nUp+nBo1HwKyygv\ -K+9sMbCp/wIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB\ -ACvYyE+PPmWFkbjyRu9LAt8DcrtyYYLRClKSg6tVvutwukLG2l//kDOohYkJtgTq\ -r6LnCIIIwYdXN+4wxugmw4cnPIZmP6kovxjhhVM95okilor1zniTAo3RN7JDIfTG\ -NgxLdGu1btt7DOFL4zTbeSJMb8M1JpPftehH+x/VLyuUCuoDMIIB5jCCAU+gAwIB\ -AgIBBTANBgkqhkiG9w0BAQsFADAhMR8wHQYDVQQDExZQYXltZW50UmVxdWVzdCBU\ -ZXN0IENBMB4XDTEzMDIyMzIyNDIyOFoXDTIzMDIyMTIyNDIyOFowPzEUMBIGA1UE\ -AwwLdGVzdGNhMS5vcmcxJzAlBgNVBAoMHlBheW1lbnQgUmVxdWVzdCBJbnRlcm1l\ -ZGlhdGUgMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAo5Vy9H3nA/OOkF5A\ -p89yfVNSiTay/LYCaB0eALpcU690U75O9Q3w2M+2AN8wpbbHsJHZMIjEeBRoQfjl\ -YXW1ucQTxWKyT+liu0D25mGXX27CBXBd4iXTxVII/iX+u3lcjORjoHOBy7QgeIDI\ -IS9y0vYu8eArpjh7m4thrVgIRtMCAwEAAaMQMA4wDAYDVR0TBAUwAwEB/zANBgkq\ -hkiG9w0BAQsFAAOBgQB9LKcVJK9sjASNzpQlpUp7nCiw5FSjVY+XMRIKK/kavzlK\ -jZ+InsmmyRVGjDoZi9GrqG9PVHgLBxi2VtVjmokZoNPqao3OfhqORAubC+JR/JLe\ -pM7aDaxDdTHVhSUk4lgNAvi26dGY7nZMsnHlPQ2tPp/HvRRiMq1oDjlylc8VTCI1\ -Eh8IsNsGEhl2qRSVYIoDyPul6RMbTSzv4tVhVfGZqIisGLLhxYsFKgxVbml0VGVz\ -dEZvdXIqgAEBE1PP93Tkpif35F+dYmXn9kLA/1djcPjCs2o2rwRMM4Uk356O5dgu\ -HXQjsfdR58qZQS9CS5DAtRUf0R8+43/wijO/hb49VNaNXmY+/cPHMkahP2aV3tZi\ -FAyZblLik9A7ZvF+UsjeFQiHB5wzWQvbqk5wQ4yabHIXoYv/E0q+eQ==\ -"; - -// -// Validly signed, but by a CA not in our root CA list -// -const char* paymentrequest5_cert1_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrxAwruAzCCAeowggFToAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAxMWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xMzA0MTkx\ -NzIwMDZaFw0yMzA0MTcxNzIwMDZaMEMxGTAXBgNVBAMMEHRlc3RtZXJjaGFudC5v\ -cmcxJjAkBgNVBAoMHVBheW1lbnQgUmVxdWVzdCBUZXN0IE1lcmNoYW50MIGfMA0G\ -CSqGSIb3DQEBAQUAA4GNADCBiQKBgQDhV6Yn47aEEmbl50YLvXoqGEJA51I/40wr\ -Z6VQGdXYaRqYktagrWDlgYY9h0JQ1bQhm8HgW7ju0R4NaDTXUqxg4HjprF0z3Mfm\ -/6mmebkLOOptfkVD7ceAteNI7cyuqWGIAZA7D9mV97mXoCAtTlBUycvkmoiClCCS\ -h0EpF/UTaQIDAQABoxAwDjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4GB\ -AGIRwW7I0QvLga+RnJoJSZNZQbtu4rQW3xmoz8WfZMBYXX3QBYg5ftycbdK+/IbP\ -qozfjGW2AS6DNArvpveSPDTK9+GJBNo1paiNtVqwXkC3Ddscv5AIms1eZGiIOQNC\ -mUvdLkpoXo48WAer3EGsZ3B15GyNEELc0q9W5yUebba1IjUSHwiw2wYSGXapFJVg\ -igPI+6XpExtNLO/i1WFV8ZmoiKwYuPvFiwUqDFVuaXRUZXN0Rml2ZSqAAXdsMgdG\ -ssymvca1S/1KeM3n8Ydi2fi1JUzAAr59xPvNJRUeqCLP9upHn5z7br3P12Oz9A20\ -5/4wL4ClPRPVnOHgij0bEg+y0tGESqmF1rfOfXDszlo2U92wCxS07kq79YAZJ1Zo\ -XYh860/Q4wvc7lfiTe+dXBzPKAKhMy91yETY\ -"; - -// -// Contains a testnet paytoaddress, so payment request network doesn't match client network -// -const char* paymentrequest1_cert2_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrQBArNBDCCAkkwggExoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAwwWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xNTAxMTEx\ -ODIxMDhaFw0yNTAxMDgxODIxMDhaMCExHzAdBgNVBAMMFlBheW1lbnRSZXF1ZXN0\ -IFRlc3QgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMsZqzkzeBGo+i2N\ -mUak3Ciodr1V7S062VOy7N0OQYNDQHYkgDFAUET7cEb5VJaHPv5m3ppTBpU9xBcf\ -wbHHUt4VjA+mhRmYrl1khjvZM+X8kEqvWn20BtcM9R6r0yIYec8UERDDHBleL/P8\ -RkxEnVLjYTV9zigCXfMsgYb3EQShAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJ\ -KoZIhvcNAQELBQADggEBABUJpl3QCqsoDSxAsQdV6zKT4VGV76AzoGj7etQsQY+r\ -+S26VfWh/fMobEzuxFChr0USgLJ6FoK78hAtoZvt1lrye9yqFv/ig3WLWsJKWHHb\ -3RT6oR03CIwZXFSUasi08QDVLxafwsU5OMcPLucF3a1lRL1ccYrNgVCCx1+X7Bos\ -tIgDGRQQ4AyoHTcfVd2hEGeUv7k14mOxFsAp6851yosHq9Q2kwmdH+rHEJbjof87\ -yyKLagc4owyXBZYkQmkeHWCNqnuRmO5vUsfVb0UUrkD64o7Th/NjwooA7SCiUXl6\ -dfygT1b7ggpx7GC+sP2DsIM47IAZ55drjqX5u2f+Ba0iPQoEdGVzdBIhCIDWwowE\ -Ehl2qRQErGqUUwSsaMpDvWIaGnJGNQqi8oisGNeMy6UFKgxKdXN0IFRlc3Rpbmcq\ -gAFwThsozZxkZxzCn4R8WxNiLFV6m0ye9fEtSbolfaW+EjBMpO03lr/dwNnrclhg\ -ew+A05xfZztrAt16XKEY7qKJ/eY2nLd0fVAIu/nIt+7/VYVXT83zLrWc150aRS7W\ -AdJbL3JOJLs6Eyp5zrPbfI8faRttFAdONKDrJgIpuW1E3g==\ -"; - -// -// Expired payment request (expires is set to 1 = 1970-01-01 00:00:01) -// -const char* paymentrequest2_cert2_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrQBArNBDCCAkkwggExoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAwwWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xNTAxMTEx\ -ODIxMDhaFw0yNTAxMDgxODIxMDhaMCExHzAdBgNVBAMMFlBheW1lbnRSZXF1ZXN0\ -IFRlc3QgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMsZqzkzeBGo+i2N\ -mUak3Ciodr1V7S062VOy7N0OQYNDQHYkgDFAUET7cEb5VJaHPv5m3ppTBpU9xBcf\ -wbHHUt4VjA+mhRmYrl1khjvZM+X8kEqvWn20BtcM9R6r0yIYec8UERDDHBleL/P8\ -RkxEnVLjYTV9zigCXfMsgYb3EQShAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJ\ -KoZIhvcNAQELBQADggEBABUJpl3QCqsoDSxAsQdV6zKT4VGV76AzoGj7etQsQY+r\ -+S26VfWh/fMobEzuxFChr0USgLJ6FoK78hAtoZvt1lrye9yqFv/ig3WLWsJKWHHb\ -3RT6oR03CIwZXFSUasi08QDVLxafwsU5OMcPLucF3a1lRL1ccYrNgVCCx1+X7Bos\ -tIgDGRQQ4AyoHTcfVd2hEGeUv7k14mOxFsAp6851yosHq9Q2kwmdH+rHEJbjof87\ -yyKLagc4owyXBZYkQmkeHWCNqnuRmO5vUsfVb0UUrkD64o7Th/NjwooA7SCiUXl6\ -dfygT1b7ggpx7GC+sP2DsIM47IAZ55drjqX5u2f+Ba0iQgoEdGVzdBIgCICt4gQS\ -GXapFASsapRTBKxoykO9YhoackY1CqLyiKwYiNLUpQUgASoQVGVzdGluZyB0ZXN0\ -bmV0ISqAATXq9A5nmJgtmee/bQTeHeif4w1YYFPBlKghwx6qbVgXTWnwBJtOQhhV\ -sZdzbTl95ENR7/Y7VJupW9kDWobCK7zUUhLAzUlwmLlcx6itHw8LTUF5HK+AwsZm\ -Zs85lISGvOS0NZW/ENa6l+oQRnL87oqVZr/EDGiuqjz6T0ThQi0l\ -"; - -// -// Unexpired payment request (expires is set to 0x7FFFFFFFFFFFFFFF = max. int64_t) -// -const char* paymentrequest3_cert2_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrQBArNBDCCAkkwggExoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAwwWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xNTAxMTEx\ -ODIxMDhaFw0yNTAxMDgxODIxMDhaMCExHzAdBgNVBAMMFlBheW1lbnRSZXF1ZXN0\ -IFRlc3QgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMsZqzkzeBGo+i2N\ -mUak3Ciodr1V7S062VOy7N0OQYNDQHYkgDFAUET7cEb5VJaHPv5m3ppTBpU9xBcf\ -wbHHUt4VjA+mhRmYrl1khjvZM+X8kEqvWn20BtcM9R6r0yIYec8UERDDHBleL/P8\ -RkxEnVLjYTV9zigCXfMsgYb3EQShAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJ\ -KoZIhvcNAQELBQADggEBABUJpl3QCqsoDSxAsQdV6zKT4VGV76AzoGj7etQsQY+r\ -+S26VfWh/fMobEzuxFChr0USgLJ6FoK78hAtoZvt1lrye9yqFv/ig3WLWsJKWHHb\ -3RT6oR03CIwZXFSUasi08QDVLxafwsU5OMcPLucF3a1lRL1ccYrNgVCCx1+X7Bos\ -tIgDGRQQ4AyoHTcfVd2hEGeUv7k14mOxFsAp6851yosHq9Q2kwmdH+rHEJbjof87\ -yyKLagc4owyXBZYkQmkeHWCNqnuRmO5vUsfVb0UUrkD64o7Th/NjwooA7SCiUXl6\ -dfygT1b7ggpx7GC+sP2DsIM47IAZ55drjqX5u2f+Ba0iSgoEdGVzdBIgCICt4gQS\ -GXapFASsapRTBKxoykO9YhoackY1CqLyiKwYyNfZpQUg//////////9/KhBUZXN0\ -aW5nIHRlc3RuZXQhKoABNwi8WnMW4aMvbmvorTiiWJLFhofLFnsoWCJnj3rWLnLh\ -n3w6q/fZ26p50ERL/noxdTUfeFsKnlECkUu/fOcOrqyYDiwvxI0SZ034DleVyFU1\ -Z3T+X0zcL8oe7bX01Yf+s2V+5JXQXarKnKBrZCGgv2ARjFNSZe7E7vGg5K4Q6Q8=\ -"; - -// -// Unexpired payment request (expires is set to 0x8000000000000000 > max. int64_t, allowed uint64) -// -const char* paymentrequest4_cert2_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrQBArNBDCCAkkwggExoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAwwWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xNTAxMTEx\ -ODIxMDhaFw0yNTAxMDgxODIxMDhaMCExHzAdBgNVBAMMFlBheW1lbnRSZXF1ZXN0\ -IFRlc3QgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMsZqzkzeBGo+i2N\ -mUak3Ciodr1V7S062VOy7N0OQYNDQHYkgDFAUET7cEb5VJaHPv5m3ppTBpU9xBcf\ -wbHHUt4VjA+mhRmYrl1khjvZM+X8kEqvWn20BtcM9R6r0yIYec8UERDDHBleL/P8\ -RkxEnVLjYTV9zigCXfMsgYb3EQShAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJ\ -KoZIhvcNAQELBQADggEBABUJpl3QCqsoDSxAsQdV6zKT4VGV76AzoGj7etQsQY+r\ -+S26VfWh/fMobEzuxFChr0USgLJ6FoK78hAtoZvt1lrye9yqFv/ig3WLWsJKWHHb\ -3RT6oR03CIwZXFSUasi08QDVLxafwsU5OMcPLucF3a1lRL1ccYrNgVCCx1+X7Bos\ -tIgDGRQQ4AyoHTcfVd2hEGeUv7k14mOxFsAp6851yosHq9Q2kwmdH+rHEJbjof87\ -yyKLagc4owyXBZYkQmkeHWCNqnuRmO5vUsfVb0UUrkD64o7Th/NjwooA7SCiUXl6\ -dfygT1b7ggpx7GC+sP2DsIM47IAZ55drjqX5u2f+Ba0iSwoEdGVzdBIgCICt4gQS\ -GXapFASsapRTBKxoykO9YhoackY1CqLyiKwYt+HZpQUggICAgICAgICAASoQVGVz\ -dGluZyB0ZXN0bmV0ISqAAXSQG8+GFA18VaKarlYrOz293rNMIub0swKGcQm8jAGX\ -HSLaRgHfUDeEPr4hydy4dtfu59KNwe2xsHOHu/SpO4L8SrA4Dm9A7SlNBVWdcLbw\ -d2hj739GDLz0b5KuJ2SG6VknMRQM976w/m2qlq0ccVGaaZ2zMIGfpzL3p6adwx/5\ -"; - -// -// Payment request with amount overflow (amount is set to 21000001 BTC) -// -const char* paymentrequest5_cert2_BASE64 = -"\ -Egt4NTA5K3NoYTI1NhrQBArNBDCCAkkwggExoAMCAQICAQEwDQYJKoZIhvcNAQEL\ -BQAwITEfMB0GA1UEAwwWUGF5bWVudFJlcXVlc3QgVGVzdCBDQTAeFw0xNTAxMTEx\ -ODIxMDhaFw0yNTAxMDgxODIxMDhaMCExHzAdBgNVBAMMFlBheW1lbnRSZXF1ZXN0\ -IFRlc3QgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMsZqzkzeBGo+i2N\ -mUak3Ciodr1V7S062VOy7N0OQYNDQHYkgDFAUET7cEb5VJaHPv5m3ppTBpU9xBcf\ -wbHHUt4VjA+mhRmYrl1khjvZM+X8kEqvWn20BtcM9R6r0yIYec8UERDDHBleL/P8\ -RkxEnVLjYTV9zigCXfMsgYb3EQShAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8wDQYJ\ -KoZIhvcNAQELBQADggEBABUJpl3QCqsoDSxAsQdV6zKT4VGV76AzoGj7etQsQY+r\ -+S26VfWh/fMobEzuxFChr0USgLJ6FoK78hAtoZvt1lrye9yqFv/ig3WLWsJKWHHb\ -3RT6oR03CIwZXFSUasi08QDVLxafwsU5OMcPLucF3a1lRL1ccYrNgVCCx1+X7Bos\ -tIgDGRQQ4AyoHTcfVd2hEGeUv7k14mOxFsAp6851yosHq9Q2kwmdH+rHEJbjof87\ -yyKLagc4owyXBZYkQmkeHWCNqnuRmO5vUsfVb0UUrkD64o7Th/NjwooA7SCiUXl6\ -dfygT1b7ggpx7GC+sP2DsIM47IAZ55drjqX5u2f+Ba0iTAoEdGVzdBIkCIDC9P+F\ -vt0DEhl2qRQErGqUUwSsaMpDvWIaGnJGNQqi8oisGLzcrKYFKhhUZXN0aW5nIGFt\ -b3VudCBvdmVyZmxvdyEqgAG8S7WEDUC6tCL6q2CTBjop/AitgEy31RL9IqYruytR\ -iEBFUrBDJZU+UEezGwr7/zoECjo5ZY3PmtZcM2sILNjyweJF6XVzGqTxUw6pN6sW\ -XR2T3Gy2LzRvhVA25QgGqpz0/juS2BtmNbsZPkN9gMMwKimgzc+PuCzmEKwPK9cQ\ -YQ==\ -"; - -#endif // BITCOIN_QT_TEST_PAYMENTREQUESTDATA_H diff --git a/src/qt/test/paymentservertests.cpp b/src/qt/test/paymentservertests.cpp deleted file mode 100644 index ef0808054d..0000000000 --- a/src/qt/test/paymentservertests.cpp +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <qt/test/paymentservertests.h> - -#include <qt/optionsmodel.h> -#include <qt/test/paymentrequestdata.h> - -#include <amount.h> -#include <interfaces/node.h> -#include <random.h> -#include <script/script.h> -#include <script/standard.h> -#include <test/setup_common.h> -#include <util/strencodings.h> - -#include <openssl/ssl.h> -#include <openssl/x509.h> -#include <openssl/x509_vfy.h> - -#include <QFileOpenEvent> -#include <QTemporaryFile> - -X509 *parse_b64der_cert(const char* cert_data) -{ - std::vector<unsigned char> data = DecodeBase64(cert_data); - assert(data.size() > 0); - const unsigned char* dptr = data.data(); - X509 *cert = d2i_X509(nullptr, &dptr, data.size()); - assert(cert); - return cert; -} - -// -// Test payment request handling -// - -static SendCoinsRecipient handleRequest(PaymentServer* server, std::vector<unsigned char>& data) -{ - RecipientCatcher sigCatcher; - QObject::connect(server, &PaymentServer::receivedPaymentRequest, - &sigCatcher, &RecipientCatcher::getRecipient); - - // Write data to a temp file: - QTemporaryFile f; - f.open(); - f.write((const char*)data.data(), data.size()); - f.close(); - - // Create a QObject, install event filter from PaymentServer - // and send a file open event to the object - QObject object; - object.installEventFilter(server); - QFileOpenEvent event(f.fileName()); - // If sending the event fails, this will cause sigCatcher to be empty, - // which will lead to a test failure anyway. - QCoreApplication::sendEvent(&object, &event); - - QObject::disconnect(server, &PaymentServer::receivedPaymentRequest, - &sigCatcher, &RecipientCatcher::getRecipient); - - // Return results from sigCatcher - return sigCatcher.recipient; -} - -void PaymentServerTests::paymentServerTests() -{ - SSL_library_init(); - BasicTestingSetup testing_setup(CBaseChainParams::MAIN); - auto node = interfaces::MakeNode(); - OptionsModel optionsModel(*node); - PaymentServer* server = new PaymentServer(nullptr, false); - X509_STORE* caStore = X509_STORE_new(); - X509_STORE_add_cert(caStore, parse_b64der_cert(caCert1_BASE64)); - PaymentServer::LoadRootCAs(caStore); - server->setOptionsModel(&optionsModel); - server->uiReady(); - - std::vector<unsigned char> data; - SendCoinsRecipient r; - QString merchant; - - // Now feed PaymentRequests to server, and observe signals it produces - - // This payment request validates directly against the - // caCert1 certificate authority: - data = DecodeBase64(paymentrequest1_cert1_BASE64); - r = handleRequest(server, data); - r.paymentRequest.getMerchant(caStore, merchant); - QCOMPARE(merchant, QString("testmerchant.org")); - - // Signed, but expired, merchant cert in the request: - data = DecodeBase64(paymentrequest2_cert1_BASE64); - r = handleRequest(server, data); - r.paymentRequest.getMerchant(caStore, merchant); - QCOMPARE(merchant, QString("")); - - // 10-long certificate chain, all intermediates valid: - data = DecodeBase64(paymentrequest3_cert1_BASE64); - r = handleRequest(server, data); - r.paymentRequest.getMerchant(caStore, merchant); - QCOMPARE(merchant, QString("testmerchant8.org")); - - // Long certificate chain, with an expired certificate in the middle: - data = DecodeBase64(paymentrequest4_cert1_BASE64); - r = handleRequest(server, data); - r.paymentRequest.getMerchant(caStore, merchant); - QCOMPARE(merchant, QString("")); - - // Validly signed, but by a CA not in our root CA list: - data = DecodeBase64(paymentrequest5_cert1_BASE64); - r = handleRequest(server, data); - r.paymentRequest.getMerchant(caStore, merchant); - QCOMPARE(merchant, QString("")); - - // Try again with no root CA's, verifiedMerchant should be empty: - caStore = X509_STORE_new(); - PaymentServer::LoadRootCAs(caStore); - data = DecodeBase64(paymentrequest1_cert1_BASE64); - r = handleRequest(server, data); - r.paymentRequest.getMerchant(caStore, merchant); - QCOMPARE(merchant, QString("")); - - // Load second root certificate - caStore = X509_STORE_new(); - X509_STORE_add_cert(caStore, parse_b64der_cert(caCert2_BASE64)); - PaymentServer::LoadRootCAs(caStore); - - QByteArray byteArray; - - // For the tests below we just need the payment request data from - // paymentrequestdata.h parsed + stored in r.paymentRequest. - // - // These tests require us to bypass the following normal client execution flow - // shown below to be able to explicitly just trigger a certain condition! - // - // handleRequest() - // -> PaymentServer::eventFilter() - // -> PaymentServer::handleURIOrFile() - // -> PaymentServer::readPaymentRequestFromFile() - // -> PaymentServer::processPaymentRequest() - - // Contains a testnet paytoaddress, so payment request network doesn't match client network: - data = DecodeBase64(paymentrequest1_cert2_BASE64); - byteArray = QByteArray((const char*)data.data(), data.size()); - r.paymentRequest.parse(byteArray); - // Ensure the request is initialized, because network "main" is default, even for - // uninitialized payment requests and that will fail our test here. - QVERIFY(r.paymentRequest.IsInitialized()); - QCOMPARE(PaymentServer::verifyNetwork(*node, r.paymentRequest.getDetails()), false); - - // Expired payment request (expires is set to 1 = 1970-01-01 00:00:01): - data = DecodeBase64(paymentrequest2_cert2_BASE64); - byteArray = QByteArray((const char*)data.data(), data.size()); - r.paymentRequest.parse(byteArray); - // Ensure the request is initialized - QVERIFY(r.paymentRequest.IsInitialized()); - // compares 1 < GetTime() == false (treated as expired payment request) - QCOMPARE(PaymentServer::verifyExpired(r.paymentRequest.getDetails()), true); - - // Unexpired payment request (expires is set to 0x7FFFFFFFFFFFFFFF = max. int64_t): - // 9223372036854775807 (uint64), 9223372036854775807 (int64_t) and -1 (int32_t) - // -1 is 1969-12-31 23:59:59 (for a 32 bit time values) - data = DecodeBase64(paymentrequest3_cert2_BASE64); - byteArray = QByteArray((const char*)data.data(), data.size()); - r.paymentRequest.parse(byteArray); - // Ensure the request is initialized - QVERIFY(r.paymentRequest.IsInitialized()); - // compares 9223372036854775807 < GetTime() == false (treated as unexpired payment request) - QCOMPARE(PaymentServer::verifyExpired(r.paymentRequest.getDetails()), false); - - // Unexpired payment request (expires is set to 0x8000000000000000 > max. int64_t, allowed uint64): - // 9223372036854775808 (uint64), -9223372036854775808 (int64_t) and 0 (int32_t) - // 0 is 1970-01-01 00:00:00 (for a 32 bit time values) - data = DecodeBase64(paymentrequest4_cert2_BASE64); - byteArray = QByteArray((const char*)data.data(), data.size()); - r.paymentRequest.parse(byteArray); - // Ensure the request is initialized - QVERIFY(r.paymentRequest.IsInitialized()); - // compares -9223372036854775808 < GetTime() == true (treated as expired payment request) - QCOMPARE(PaymentServer::verifyExpired(r.paymentRequest.getDetails()), true); - - // Test BIP70 DoS protection: - auto randdata = FastRandomContext().randbytes(BIP70_MAX_PAYMENTREQUEST_SIZE + 1); - - // Write data to a temp file: - QTemporaryFile tempFile; - tempFile.open(); - tempFile.write((const char*)randdata.data(), randdata.size()); - tempFile.close(); - // compares 50001 <= BIP70_MAX_PAYMENTREQUEST_SIZE == false - QCOMPARE(PaymentServer::verifySize(tempFile.size()), false); - - // Payment request with amount overflow (amount is set to 21000001 BTC): - data = DecodeBase64(paymentrequest5_cert2_BASE64); - byteArray = QByteArray((const char*)data.data(), data.size()); - r.paymentRequest.parse(byteArray); - // Ensure the request is initialized - QVERIFY(r.paymentRequest.IsInitialized()); - // Extract address and amount from the request - QList<std::pair<CScript, CAmount> > sendingTos = r.paymentRequest.getPayTo(); - for (const std::pair<CScript, CAmount>& sendingTo : sendingTos) { - CTxDestination dest; - if (ExtractDestination(sendingTo.first, dest)) - QCOMPARE(PaymentServer::verifyAmount(sendingTo.second), false); - } - - delete server; -} - -void RecipientCatcher::getRecipient(const SendCoinsRecipient& r) -{ - recipient = r; -} diff --git a/src/qt/test/paymentservertests.h b/src/qt/test/paymentservertests.h deleted file mode 100644 index 7ef7a0a641..0000000000 --- a/src/qt/test/paymentservertests.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H -#define BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H - -#include <qt/paymentserver.h> - -#include <QObject> -#include <QTest> - -class PaymentServerTests : public QObject -{ - Q_OBJECT - -private Q_SLOTS: - void paymentServerTests(); -}; - -// Dummy class to receive paymentserver signals. -// If SendCoinsRecipient was a proper QObject, then -// we could use QSignalSpy... but it's not. -class RecipientCatcher : public QObject -{ - Q_OBJECT - -public Q_SLOTS: - void getRecipient(const SendCoinsRecipient& r); - -public: - SendCoinsRecipient recipient; -}; - -#endif // BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 3c2ffa6c00..971d9f4a7c 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -7,7 +7,7 @@ #include <interfaces/node.h> #include <rpc/server.h> #include <qt/rpcconsole.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <univalue.h> #include <util/system.h> @@ -41,7 +41,7 @@ void RPCNestedTests::rpcNestedTests() std::string result; std::string result2; std::string filtered; - auto node = interfaces::MakeNode(); + interfaces::Node* node = &m_node; RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo()[chain]", &filtered); //simple result filtering with path QVERIFY(result=="main"); QVERIFY(filtered == "getblockchaininfo()[chain]"); diff --git a/src/qt/test/rpcnestedtests.h b/src/qt/test/rpcnestedtests.h index 97143ff78a..8789fe8373 100644 --- a/src/qt/test/rpcnestedtests.h +++ b/src/qt/test/rpcnestedtests.h @@ -8,8 +8,16 @@ #include <QObject> #include <QTest> +namespace interfaces { +class Node; +} // namespace interfaces + class RPCNestedTests : public QObject { +public: + RPCNestedTests(interfaces::Node& node) : m_node(node) {} + interfaces::Node& m_node; + Q_OBJECT private Q_SLOTS: diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index c39266a397..243c10d7da 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -12,13 +12,10 @@ #include <qt/test/rpcnestedtests.h> #include <qt/test/uritests.h> #include <qt/test/compattests.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #ifdef ENABLE_WALLET #include <qt/test/addressbooktests.h> -#ifdef ENABLE_BIP70 -#include <qt/test/paymentservertests.h> -#endif // ENABLE_BIP70 #include <qt/test/wallettests.h> #endif // ENABLE_WALLET @@ -53,7 +50,7 @@ int main(int argc, char *argv[]) BasicTestingSetup dummy{CBaseChainParams::REGTEST}; } - auto node = interfaces::MakeNode(); + std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(); bool fInvalid = false; @@ -79,13 +76,7 @@ int main(int argc, char *argv[]) if (QTest::qExec(&test1) != 0) { fInvalid = true; } -#if defined(ENABLE_WALLET) && defined(ENABLE_BIP70) - PaymentServerTests test2; - if (QTest::qExec(&test2) != 0) { - fInvalid = true; - } -#endif - RPCNestedTests test3; + RPCNestedTests test3(*node); if (QTest::qExec(&test3) != 0) { fInvalid = true; } @@ -94,11 +85,11 @@ int main(int argc, char *argv[]) fInvalid = true; } #ifdef ENABLE_WALLET - WalletTests test5; + WalletTests test5(*node); if (QTest::qExec(&test5) != 0) { fInvalid = true; } - AddressBookTests test6; + AddressBookTests test6(*node); if (QTest::qExec(&test6) != 0) { fInvalid = true; } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index eea874c0d4..980de711db 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -13,7 +13,7 @@ #include <qt/transactionview.h> #include <qt/walletmodel.h> #include <key_io.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <validation.h> #include <wallet/wallet.h> #include <qt/overviewpage.h> @@ -126,21 +126,25 @@ void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, st // QT_QPA_PLATFORM=xcb src/qt/test/test_bitcoin-qt # Linux // QT_QPA_PLATFORM=windows src/qt/test/test_bitcoin-qt # Windows // QT_QPA_PLATFORM=cocoa src/qt/test/test_bitcoin-qt # macOS -void TestGUI() +void TestGUI(interfaces::Node& node) { // Set up wallet and chain with 105 blocks (5 mature blocks for spending). TestChain100Setup test; for (int i = 0; i < 5; ++i) { test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); } - auto chain = interfaces::MakeChain(); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + node.context()->connman = std::move(test.m_node.connman); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); { + auto spk_man = wallet->GetLegacyScriptPubKeyMan(); + auto locked_chain = wallet->chain().lock(); LOCK(wallet->cs_wallet); + AssertLockHeld(spk_man->cs_wallet); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); - wallet->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { auto locked_chain = wallet->chain().lock(); @@ -159,10 +163,9 @@ void TestGUI() std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); - auto node = interfaces::MakeNode(); - OptionsModel optionsModel(*node); + OptionsModel optionsModel(node); AddWallet(wallet); - WalletModel walletModel(std::move(node->getWallets().back()), *node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); RemoveWallet(wallet); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); @@ -260,5 +263,5 @@ void WalletTests::walletTests() return; } #endif - TestGUI(); + TestGUI(m_node); } diff --git a/src/qt/test/wallettests.h b/src/qt/test/wallettests.h index 342f7916c3..0a7b57a678 100644 --- a/src/qt/test/wallettests.h +++ b/src/qt/test/wallettests.h @@ -4,8 +4,16 @@ #include <QObject> #include <QTest> +namespace interfaces { +class Node; +} // namespace interfaces + class WalletTests : public QObject { + public: + WalletTests(interfaces::Node& node) : m_node(node) {} + interfaces::Node& m_node; + Q_OBJECT private Q_SLOTS: diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 44ce5265cd..318b0756c7 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -48,7 +48,6 @@ QString TransactionDesc::FormatTxStatus(const interfaces::WalletTx& wtx, const i } } -#ifndef ENABLE_BIP70 // Takes an encoded PaymentRequest as a string and tries to find the Common Name of the X.509 certificate // used to sign the PaymentRequest. bool GetPaymentRequestMerchant(const std::string& pr, QString& merchant) @@ -76,7 +75,6 @@ bool GetPaymentRequestMerchant(const std::string& pr, QString& merchant) } return false; } -#endif QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wallet, TransactionRecord *rec, int unit) { @@ -294,19 +292,11 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall if (r.first == "PaymentRequest") { QString merchant; -#ifdef ENABLE_BIP70 - PaymentRequestPlus req; - req.parse(QByteArray::fromRawData(r.second.data(), r.second.size())); - if (!req.getMerchant(PaymentServer::getCertStore(), merchant)) { - merchant.clear(); - } -#else if (!GetPaymentRequestMerchant(r.second, merchant)) { merchant.clear(); } else { merchant += tr(" (Certificate was not verified)"); } -#endif if (!merchant.isNull()) { strHTML += "<b>" + tr("Merchant") + ":</b> " + GUIUtil::HtmlEscape(merchant) + "<br>"; } diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 6509a701f3..095c98d26f 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -11,9 +11,6 @@ #include <qt/forms/ui_helpmessagedialog.h> #include <qt/bitcoingui.h> -#ifdef ENABLE_BIP70 -#include <qt/paymentrequestplus.h> -#endif #include <clientversion.h> #include <init.h> diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index a7edf442e5..4c253f8ddd 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -109,6 +109,12 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal wallet_model->setParent(this); m_wallets.push_back(wallet_model); + // WalletModel::startPollBalance needs to be called in a thread managed by + // Qt because of startTimer. Considering the current thread can be a RPC + // thread, better delegate the calling to Qt with Qt::AutoConnection. + const bool called = QMetaObject::invokeMethod(wallet_model, "startPollBalance"); + assert(called); + connect(wallet_model, &WalletModel::unload, [this, wallet_model] { // Defer removeAndDeleteWallet when no modal widget is active. // TODO: remove this workaround by removing usage of QDiallog::exec. diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index d7f0617315..4b2b475883 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -61,11 +61,6 @@ void WalletFrame::addWallet(WalletModel *walletModel) walletStack->addWidget(walletView); mapWalletViews[walletModel] = walletView; - // Ensure a walletView is able to show the main window - connect(walletView, &WalletView::showNormalIfMinimized, [this]{ - gui->showNormalIfMinimized(); - }); - connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 49a13330ec..33801d3907 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -44,11 +44,6 @@ WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces: transactionTableModel = new TransactionTableModel(platformStyle, this); recentRequestsTableModel = new RecentRequestsTableModel(this); - // This timer will be fired repeatedly to update the balance - pollTimer = new QTimer(this); - connect(pollTimer, &QTimer::timeout, this, &WalletModel::pollBalanceChanged); - pollTimer->start(MODEL_UPDATE_DELAY); - subscribeToCoreSignals(); } @@ -57,6 +52,14 @@ WalletModel::~WalletModel() unsubscribeFromCoreSignals(); } +void WalletModel::startPollBalance() +{ + // This timer will be fired repeatedly to update the balance + QTimer* timer = new QTimer(this); + connect(timer, &QTimer::timeout, this, &WalletModel::pollBalanceChanged); + timer->start(MODEL_UPDATE_DELAY); +} + void WalletModel::updateStatus() { EncryptionStatus newEncryptionStatus = getEncryptionStatus(); @@ -143,31 +146,6 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { if (rcp.fSubtractFeeFromAmount) fSubtractFeeFromAmount = true; - -#ifdef ENABLE_BIP70 - if (rcp.paymentRequest.IsInitialized()) - { // PaymentRequest... - CAmount subtotal = 0; - const payments::PaymentDetails& details = rcp.paymentRequest.getDetails(); - for (int i = 0; i < details.outputs_size(); i++) - { - const payments::Output& out = details.outputs(i); - if (out.amount() <= 0) continue; - subtotal += out.amount(); - const unsigned char* scriptStr = (const unsigned char*)out.script().data(); - CScript scriptPubKey(scriptStr, scriptStr+out.script().size()); - CAmount nAmount = out.amount(); - CRecipient recipient = {scriptPubKey, nAmount, rcp.fSubtractFeeFromAmount}; - vecSend.push_back(recipient); - } - if (subtotal <= 0) - { - return InvalidAmount; - } - total += subtotal; - } - else -#endif { // User-entered bitcoin address / amount: if(!validateAddress(rcp.address)) { @@ -240,29 +218,12 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran std::vector<std::pair<std::string, std::string>> vOrderForm; for (const SendCoinsRecipient &rcp : transaction.getRecipients()) { -#ifdef ENABLE_BIP70 - if (rcp.paymentRequest.IsInitialized()) - { - // Make sure any payment requests involved are still valid. - if (PaymentServer::verifyExpired(rcp.paymentRequest.getDetails())) { - return PaymentRequestExpired; - } - - // Store PaymentRequests in wtx.vOrderForm in wallet. - std::string value; - rcp.paymentRequest.SerializeToString(&value); - vOrderForm.emplace_back("PaymentRequest", std::move(value)); - } - else -#endif if (!rcp.message.isEmpty()) // Message from normal bitcoin:URI (bitcoin:123...?message=example) vOrderForm.emplace_back("Message", rcp.message.toStdString()); } auto& newTx = transaction.getWtx(); - std::string rejectReason; - if (!wallet().commitTransaction(newTx, {} /* mapValue */, std::move(vOrderForm), rejectReason)) - return SendCoinsReturn(TransactionCommitFailed, QString::fromStdString(rejectReason)); + wallet().commitTransaction(newTx, {} /* mapValue */, std::move(vOrderForm)); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << *newTx; @@ -273,10 +234,6 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran // and emit coinsSent signal for each recipient for (const SendCoinsRecipient &rcp : transaction.getRecipients()) { - // Don't touch the address book when we have a payment request -#ifdef ENABLE_BIP70 - if (!rcp.paymentRequest.IsInitialized()) -#endif { std::string strAddress = rcp.address.toStdString(); CTxDestination dest = DecodeDestination(strAddress); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index f16fe81bd9..a873519a34 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -5,18 +5,15 @@ #ifndef BITCOIN_QT_WALLETMODEL_H #define BITCOIN_QT_WALLETMODEL_H +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <amount.h> #include <key.h> #include <serialize.h> #include <script/standard.h> -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#ifdef ENABLE_BIP70 -#include <qt/paymentrequestplus.h> -#endif #include <qt/walletmodeltransaction.h> #include <interfaces/wallet.h> @@ -67,15 +64,9 @@ public: CAmount amount; // If from a payment request, this is used for storing the memo QString message; - -#ifdef ENABLE_BIP70 - // If from a payment request, paymentRequest.IsInitialized() will be true - PaymentRequestPlus paymentRequest; -#else - // If building with BIP70 is disabled, keep the payment request around as - // serialized string to ensure load/store is lossless + // Keep the payment request around as a serialized string to ensure + // load/store is lossless. std::string sPaymentRequest; -#endif // Empty if no authentication or invalid signature/cert/etc. QString authenticatedMerchant; @@ -91,11 +82,6 @@ public: std::string sAddress = address.toStdString(); std::string sLabel = label.toStdString(); std::string sMessage = message.toStdString(); -#ifdef ENABLE_BIP70 - std::string sPaymentRequest; - if (!ser_action.ForRead() && paymentRequest.IsInitialized()) - paymentRequest.SerializeToString(&sPaymentRequest); -#endif std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); READWRITE(this->nVersion); @@ -111,10 +97,6 @@ public: address = QString::fromStdString(sAddress); label = QString::fromStdString(sLabel); message = QString::fromStdString(sMessage); -#ifdef ENABLE_BIP70 - if (!sPaymentRequest.empty()) - paymentRequest.parse(QByteArray::fromRawData(sPaymentRequest.data(), sPaymentRequest.size())); -#endif authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); } } @@ -138,7 +120,6 @@ public: AmountWithFeeExceedsBalance, DuplicateAddress, TransactionCreationFailed, // Error returned when wallet is still locked - TransactionCommitFailed, AbsurdFee, PaymentRequestExpired }; @@ -254,8 +235,6 @@ private: EncryptionStatus cachedEncryptionStatus; int cachedNumBlocks; - QTimer *pollTimer; - void subscribeToCoreSignals(); void unsubscribeFromCoreSignals(); void checkBalanceChanged(const interfaces::WalletBalances& new_balances); @@ -291,6 +270,9 @@ Q_SIGNALS: void canGetAddressesChanged(); public Q_SLOTS: + /* Starts a timer to periodically update the balance */ + void startPollBalance(); + /* Wallet status might have changed */ void updateStatus(); /* New transaction, or transaction changed status */ diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index d00ccf70d9..eba95bd27c 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -48,25 +48,6 @@ void WalletModelTransaction::reassignAmounts(int nChangePosRet) for (QList<SendCoinsRecipient>::iterator it = recipients.begin(); it != recipients.end(); ++it) { SendCoinsRecipient& rcp = (*it); - -#ifdef ENABLE_BIP70 - if (rcp.paymentRequest.IsInitialized()) - { - CAmount subtotal = 0; - const payments::PaymentDetails& details = rcp.paymentRequest.getDetails(); - for (int j = 0; j < details.outputs_size(); j++) - { - const payments::Output& out = details.outputs(j); - if (out.amount() <= 0) continue; - if (i == nChangePosRet) - i++; - subtotal += walletTransaction->vout[i].nValue; - i++; - } - rcp.amount = subtotal; - } - else // normal recipient (no payment request) -#endif { if (i == nChangePosRet) i++; diff --git a/src/qt/walletview.h b/src/qt/walletview.h index e29c4c52f5..8d5a301cdb 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -115,8 +115,6 @@ public Q_SLOTS: void requestedSyncWarningInfo(); Q_SIGNALS: - /** Signal that we want to show the main window */ - void showNormalIfMinimized(); /** Fired when a message should be reported to the user */ void message(const QString &title, const QString &message, unsigned int style); /** Encryption status of wallet changed */ diff --git a/src/random.cpp b/src/random.cpp index 48d20d7d72..3e6398f7b4 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -5,19 +5,22 @@ #include <random.h> +#include <compat/cpuid.h> #include <crypto/sha512.h> #include <support/cleanse.h> #ifdef WIN32 #include <compat.h> // for Windows API #include <wincrypt.h> #endif -#include <logging.h> // for LogPrint() -#include <sync.h> // for WAIT_LOCK +#include <logging.h> // for LogPrintf() +#include <sync.h> // for Mutex #include <util/time.h> // for GetTime() #include <stdlib.h> #include <thread> +#include <randomenv.h> + #include <support/allocators/secure.h> #ifndef WIN32 @@ -40,11 +43,6 @@ #include <sys/sysctl.h> #endif - -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -#include <cpuid.h> -#endif - #include <openssl/err.h> #include <openssl/rand.h> #include <openssl/conf.h> @@ -75,7 +73,7 @@ static inline int64_t GetPerformanceCounter() noexcept #endif } -#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#ifdef HAVE_GETCPUID static bool g_rdrand_supported = false; static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; @@ -86,15 +84,6 @@ static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND" #ifdef bit_RDSEED static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); #endif -static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ - // We can't use __get_cpuid as it doesn't support subleafs. -#ifdef __GNUC__ - __cpuid_count(leaf, subleaf, a, b, c, d); -#else - __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); -#endif -} static void InitHardwareRand() { @@ -263,44 +252,6 @@ static void Strengthen(const unsigned char (&seed)[32], int microseconds, CSHA51 memory_cleanse(buffer, sizeof(buffer)); } -static void RandAddSeedPerfmon(CSHA512& hasher) -{ -#ifdef WIN32 - // Don't need this on Linux, OpenSSL automatically uses /dev/urandom - // Seed with the entire set of perfmon data - - // This can take up to 2 seconds, so only do it every 10 minutes - static int64_t nLastPerfmon; - if (GetTime() < nLastPerfmon + 10 * 60) - return; - nLastPerfmon = GetTime(); - - std::vector<unsigned char> vData(250000, 0); - long ret = 0; - unsigned long nSize = 0; - const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data - while (true) { - nSize = vData.size(); - ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); - if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) - break; - vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially - } - RegCloseKey(HKEY_PERFORMANCE_DATA); - if (ret == ERROR_SUCCESS) { - hasher.Write(vData.data(), nSize); - memory_cleanse(vData.data(), nSize); - } else { - // Performance data is only a best-effort attempt at improving the - // situation when the OS randomness (and other sources) aren't - // adequate. As a result, failure to read it is isn't considered critical, - // so we don't call RandFailure(). - // TODO: Add logging when the logger is made functional before global - // constructors have been invoked. - } -#endif -} - #ifndef WIN32 /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. @@ -556,22 +507,16 @@ static void SeedSlow(CSHA512& hasher) noexcept } /** Extract entropy from rng, strengthen it, and feed it into hasher. */ -static void SeedStrengthen(CSHA512& hasher, RNGState& rng) noexcept +static void SeedStrengthen(CSHA512& hasher, RNGState& rng, int microseconds) noexcept { - static std::atomic<int64_t> last_strengthen{0}; - int64_t last_time = last_strengthen.load(); - int64_t current_time = GetTimeMicros(); - if (current_time > last_time + 60000000) { // Only run once a minute - // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. - unsigned char strengthen_seed[32]; - rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); - // Strengthen it for 10ms (100ms on first run), and feed it into hasher. - Strengthen(strengthen_seed, last_time == 0 ? 100000 : 10000, hasher); - last_strengthen = current_time; - } + // Generate 32 bytes of entropy from the RNG, and a copy of the entropy already in hasher. + unsigned char strengthen_seed[32]; + rng.MixExtract(strengthen_seed, sizeof(strengthen_seed), CSHA512(hasher), false); + // Strengthen the seed, and feed it into hasher. + Strengthen(strengthen_seed, microseconds, hasher); } -static void SeedSleep(CSHA512& hasher, RNGState& rng) +static void SeedPeriodic(CSHA512& hasher, RNGState& rng) { // Everything that the 'fast' seeder includes SeedFast(hasher); @@ -579,17 +524,13 @@ static void SeedSleep(CSHA512& hasher, RNGState& rng) // High-precision timestamp SeedTimestamp(hasher); - // Sleep for 1ms - MilliSleep(1); - - // High-precision timestamp after sleeping (as we commit to both the time before and after, this measures the delay) - SeedTimestamp(hasher); - - // Windows performance monitor data (once every 10 minutes) - RandAddSeedPerfmon(hasher); + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); + LogPrintf("Feeding %i bytes of dynamic environment data into RNG\n", hasher.Size() - old_size); - // Strengthen every minute - SeedStrengthen(hasher, rng); + // Strengthen for 10 ms + SeedStrengthen(hasher, rng, 10000); } static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept @@ -600,17 +541,22 @@ static void SeedStartup(CSHA512& hasher, RNGState& rng) noexcept // Everything that the 'slow' seeder includes. SeedSlow(hasher); - // Windows performance monitor data. - RandAddSeedPerfmon(hasher); + // Dynamic environment data (performance monitoring, ...) + auto old_size = hasher.Size(); + RandAddDynamicEnv(hasher); + + // Static environment data + RandAddStaticEnv(hasher); + LogPrintf("Feeding %i bytes of environment data into RNG\n", hasher.Size() - old_size); - // Strengthen - SeedStrengthen(hasher, rng); + // Strengthen for 100 ms + SeedStrengthen(hasher, rng, 100000); } enum class RNGLevel { FAST, //!< Automatically called by GetRandBytes SLOW, //!< Automatically called by GetStrongRandBytes - SLEEP, //!< Called by RandAddSeedSleep() + PERIODIC, //!< Called by RandAddPeriodic() }; static void ProcRand(unsigned char* out, int num, RNGLevel level) @@ -628,8 +574,8 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) case RNGLevel::SLOW: SeedSlow(hasher); break; - case RNGLevel::SLEEP: - SeedSleep(hasher, rng); + case RNGLevel::PERIODIC: + SeedPeriodic(hasher, rng); break; } @@ -652,7 +598,7 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } -void RandAddSeedSleep() { ProcRand(nullptr, 0, RNGLevel::SLEEP); } +void RandAddPeriodic() { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } bool g_mock_deterministic_tests{false}; @@ -716,7 +662,7 @@ bool Random_SanityCheck() uint64_t start = GetPerformanceCounter(); /* This does not measure the quality of randomness, but it does test that - * OSRandom() overwrites all 32 bytes of the output given a maximum + * GetOSRand() overwrites all 32 bytes of the output given a maximum * number of tries. */ static const ssize_t MAX_TRIES = 1024; diff --git a/src/random.h b/src/random.h index 22801ec155..7768f9d3c5 100644 --- a/src/random.h +++ b/src/random.h @@ -52,7 +52,6 @@ * sources used in the 'slow' seeder are included, but also: * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. - * - (On Windows) Through OpenSSL, the screen contents. * - Strengthen the entropy for 100 ms using repeated SHA512. * * When mixing in new entropy, H = SHA512(entropy || old_rng_state) is computed, and @@ -85,11 +84,11 @@ uint256 GetRandHash() noexcept; void GetStrongRandBytes(unsigned char* buf, int num) noexcept; /** - * Sleep for 1ms, gather entropy from various sources, and feed them to the PRNG state. + * Gather entropy from various expensive sources, and feed them to the PRNG state. * * Thread-safe. */ -void RandAddSeedSleep(); +void RandAddPeriodic(); /** * Fast randomness source. This is seeded once with secure random data, but @@ -166,6 +165,7 @@ public: /** Generate a random integer in the range [0..range). */ uint64_t randrange(uint64_t range) noexcept { + assert(range); --range; int bits = CountBits(range); while (true) { diff --git a/src/randomenv.cpp b/src/randomenv.cpp new file mode 100644 index 0000000000..603c88eaab --- /dev/null +++ b/src/randomenv.cpp @@ -0,0 +1,508 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <randomenv.h> + +#include <clientversion.h> +#include <compat/cpuid.h> +#include <crypto/sha512.h> +#include <support/cleanse.h> +#include <util/time.h> // for GetTime() +#ifdef WIN32 +#include <compat.h> // for Windows API +#endif + +#include <algorithm> +#include <atomic> +#include <chrono> +#include <climits> +#include <thread> +#include <vector> + +#include <stdint.h> +#include <string.h> +#ifndef WIN32 +#include <sys/types.h> // must go before a number of other headers +#include <fcntl.h> +#include <netinet/in.h> +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/utsname.h> +#include <unistd.h> +#endif +#ifdef __MACH__ +#include <mach/clock.h> +#include <mach/mach.h> +#include <mach/mach_time.h> +#endif +#if HAVE_DECL_GETIFADDRS +#include <ifaddrs.h> +#endif +#if HAVE_SYSCTL +#include <sys/sysctl.h> +#if HAVE_VM_VM_PARAM_H +#include <vm/vm_param.h> +#endif +#if HAVE_SYS_RESOURCES_H +#include <sys/resources.h> +#endif +#if HAVE_SYS_VMMETER_H +#include <sys/vmmeter.h> +#endif +#endif +#ifdef __linux__ +#include <sys/auxv.h> +#endif + +//! Necessary on some platforms +extern char** environ; + +namespace { + +void RandAddSeedPerfmon(CSHA512& hasher) +{ +#ifdef WIN32 + // Don't need this on Linux, OpenSSL automatically uses /dev/urandom + // Seed with the entire set of perfmon data + + // This can take up to 2 seconds, so only do it every 10 minutes + static std::atomic<std::chrono::seconds> last_perfmon{std::chrono::seconds{0}}; + auto last_time = last_perfmon.load(); + auto current_time = GetTime<std::chrono::seconds>(); + if (current_time < last_time + std::chrono::minutes{10}) return; + last_perfmon = current_time; + + std::vector<unsigned char> vData(250000, 0); + long ret = 0; + unsigned long nSize = 0; + const size_t nMaxSize = 10000000; // Bail out at more than 10MB of performance data + while (true) { + nSize = vData.size(); + ret = RegQueryValueExA(HKEY_PERFORMANCE_DATA, "Global", nullptr, nullptr, vData.data(), &nSize); + if (ret != ERROR_MORE_DATA || vData.size() >= nMaxSize) + break; + vData.resize(std::max((vData.size() * 3) / 2, nMaxSize)); // Grow size of buffer exponentially + } + RegCloseKey(HKEY_PERFORMANCE_DATA); + if (ret == ERROR_SUCCESS) { + hasher.Write(vData.data(), nSize); + memory_cleanse(vData.data(), nSize); + } else { + // Performance data is only a best-effort attempt at improving the + // situation when the OS randomness (and other sources) aren't + // adequate. As a result, failure to read it is isn't considered critical, + // so we don't call RandFailure(). + // TODO: Add logging when the logger is made functional before global + // constructors have been invoked. + } +#endif +} + +/** Helper to easily feed data into a CSHA512. + * + * Note that this does not serialize the passed object (like stream.h's << operators do). + * Its raw memory representation is used directly. + */ +template<typename T> +CSHA512& operator<<(CSHA512& hasher, const T& data) { + static_assert(!std::is_same<typename std::decay<T>::type, char*>::value, "Calling operator<<(CSHA512, char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, unsigned char*>::value, "Calling operator<<(CSHA512, unsigned char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const char*>::value, "Calling operator<<(CSHA512, const char*) is probably not what you want"); + static_assert(!std::is_same<typename std::decay<T>::type, const unsigned char*>::value, "Calling operator<<(CSHA512, const unsigned char*) is probably not what you want"); + hasher.Write((const unsigned char*)&data, sizeof(data)); + return hasher; +} + +#ifndef WIN32 +void AddSockaddr(CSHA512& hasher, const struct sockaddr *addr) +{ + if (addr == nullptr) return; + switch (addr->sa_family) { + case AF_INET: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in)); + break; + case AF_INET6: + hasher.Write((const unsigned char*)addr, sizeof(sockaddr_in6)); + break; + default: + hasher.Write((const unsigned char*)&addr->sa_family, sizeof(addr->sa_family)); + } +} + +void AddFile(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + int f = open(path, O_RDONLY); + size_t total = 0; + if (f != -1) { + unsigned char fbuf[4096]; + int n; + hasher.Write((const unsigned char*)&f, sizeof(f)); + if (fstat(f, &sb) == 0) hasher << sb; + do { + n = read(f, fbuf, sizeof(fbuf)); + if (n > 0) hasher.Write(fbuf, n); + total += n; + /* not bothering with EINTR handling. */ + } while (n == sizeof(fbuf) && total < 1048576); // Read only the first 1 Mbyte + close(f); + } +} + +void AddPath(CSHA512& hasher, const char *path) +{ + struct stat sb = {}; + if (stat(path, &sb) == 0) { + hasher.Write((const unsigned char*)path, strlen(path) + 1); + hasher << sb; + } +} +#endif + +#if HAVE_SYSCTL +template<int... S> +void AddSysctl(CSHA512& hasher) +{ + int CTL[sizeof...(S)] = {S...}; + unsigned char buffer[65536]; + size_t siz = 65536; + int ret = sysctl(CTL, sizeof...(S), buffer, &siz, nullptr, 0); + if (ret == 0 || (ret == -1 && errno == ENOMEM)) { + hasher << sizeof(CTL); + hasher.Write((const unsigned char*)CTL, sizeof(CTL)); + if (siz > sizeof(buffer)) siz = sizeof(buffer); + hasher << siz; + hasher.Write(buffer, siz); + } +} +#endif + +#ifdef HAVE_GETCPUID +void inline AddCPUID(CSHA512& hasher, uint32_t leaf, uint32_t subleaf, uint32_t& ax, uint32_t& bx, uint32_t& cx, uint32_t& dx) +{ + GetCPUID(leaf, subleaf, ax, bx, cx, dx); + hasher << leaf << subleaf << ax << bx << cx << dx; +} + +void AddAllCPUID(CSHA512& hasher) +{ + uint32_t ax, bx, cx, dx; + // Iterate over all standard leaves + AddCPUID(hasher, 0, 0, ax, bx, cx, dx); // Returns max leaf in ax + uint32_t max = ax; + for (uint32_t leaf = 1; leaf <= max; ++leaf) { + for (uint32_t subleaf = 0;; ++subleaf) { + AddCPUID(hasher, leaf, subleaf, ax, bx, cx, dx); + // Iterate over subleaves for leaf 4, 11, 13 + if (leaf != 4 && leaf != 11 && leaf != 13) break; + if ((leaf == 4 || leaf == 13) && ax == 0) break; + if (leaf == 11 && (cx & 0xFF00) == 0) break; + } + } + // Iterate over all extended leaves + AddCPUID(hasher, 0x80000000, 0, ax, bx, cx, dx); // Returns max extended leaf in ax + uint32_t ext_max = ax; + for (uint32_t leaf = 0x80000001; leaf <= ext_max; ++leaf) { + AddCPUID(hasher, leaf, 0, ax, bx, cx, dx); + } +} +#endif +} // namespace + +void RandAddDynamicEnv(CSHA512& hasher) +{ + RandAddSeedPerfmon(hasher); + + // Various clocks +#ifdef WIN32 + FILETIME ftime; + GetSystemTimeAsFileTime(&ftime); + hasher << ftime; +#else +# ifndef __MACH__ + // On non-MacOS systems, use various clock_gettime() calls. + struct timespec ts = {}; +# ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &ts); + hasher << ts; +# endif +# ifdef CLOCK_REALTIME + clock_gettime(CLOCK_REALTIME, &ts); + hasher << ts; +# endif +# ifdef CLOCK_BOOTTIME + clock_gettime(CLOCK_BOOTTIME, &ts); + hasher << ts; +# endif +# else + // On MacOS use mach_absolute_time (number of CPU ticks since boot) as a replacement for CLOCK_MONOTONIC, + // and clock_get_time for CALENDAR_CLOCK as a replacement for CLOCK_REALTIME. + hasher << mach_absolute_time(); + // From https://gist.github.com/jbenet/1087739 + clock_serv_t cclock; + mach_timespec_t mts = {}; + if (host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock) == KERN_SUCCESS && clock_get_time(cclock, &mts) == KERN_SUCCESS) { + hasher << mts; + mach_port_deallocate(mach_task_self(), cclock); + } +# endif + // gettimeofday is available on all UNIX systems, but only has microsecond precision. + struct timeval tv = {}; + gettimeofday(&tv, nullptr); + hasher << tv; +#endif + // Probably redundant, but also use all the clocks C++11 provides: + hasher << std::chrono::system_clock::now().time_since_epoch().count(); + hasher << std::chrono::steady_clock::now().time_since_epoch().count(); + hasher << std::chrono::high_resolution_clock::now().time_since_epoch().count(); + +#ifndef WIN32 + // Current resource usage. + struct rusage usage = {}; + if (getrusage(RUSAGE_SELF, &usage) == 0) hasher << usage; +#endif + +#ifdef __linux__ + AddFile(hasher, "/proc/diskstats"); + AddFile(hasher, "/proc/vmstat"); + AddFile(hasher, "/proc/schedstat"); + AddFile(hasher, "/proc/zoneinfo"); + AddFile(hasher, "/proc/meminfo"); + AddFile(hasher, "/proc/softirqs"); + AddFile(hasher, "/proc/stat"); + AddFile(hasher, "/proc/self/schedstat"); + AddFile(hasher, "/proc/self/status"); +#endif + +#if HAVE_SYSCTL +# ifdef CTL_KERN +# if defined(KERN_PROC) && defined(KERN_PROC_ALL) + AddSysctl<CTL_KERN, KERN_PROC, KERN_PROC_ALL>(hasher); +# endif +# endif +# ifdef CTL_HW +# ifdef HW_DISKSTATS + AddSysctl<CTL_HW, HW_DISKSTATS>(hasher); +# endif +# endif +# ifdef CTL_VM +# ifdef VM_LOADAVG + AddSysctl<CTL_VM, VM_LOADAVG>(hasher); +# endif +# ifdef VM_TOTAL + AddSysctl<CTL_VM, VM_TOTAL>(hasher); +# endif +# ifdef VM_METER + AddSysctl<CTL_VM, VM_METER>(hasher); +# endif +# endif +#endif + + // Stack and heap location + void* addr = malloc(4097); + hasher << &addr << addr; + free(addr); +} + +void RandAddStaticEnv(CSHA512& hasher) +{ + // Some compile-time static properties + hasher << (CHAR_MIN < 0) << sizeof(void*) << sizeof(long) << sizeof(int); +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + hasher << __GNUC__ << __GNUC_MINOR__ << __GNUC_PATCHLEVEL__; +#endif +#ifdef _MSC_VER + hasher << _MSC_VER; +#endif + hasher << __cplusplus; +#ifdef _XOPEN_VERSION + hasher << _XOPEN_VERSION; +#endif +#ifdef __VERSION__ + const char* COMPILER_VERSION = __VERSION__; + hasher.Write((const unsigned char*)COMPILER_VERSION, strlen(COMPILER_VERSION) + 1); +#endif + + // Bitcoin client version + hasher << CLIENT_VERSION; + +#ifdef __linux__ + // Information available through getauxval() +# ifdef AT_HWCAP + hasher << getauxval(AT_HWCAP); +# endif +# ifdef AT_HWCAP2 + hasher << getauxval(AT_HWCAP2); +# endif +# ifdef AT_RANDOM + const unsigned char* random_aux = (const unsigned char*)getauxval(AT_RANDOM); + if (random_aux) hasher.Write(random_aux, 16); +# endif +# ifdef AT_PLATFORM + const char* platform_str = (const char*)getauxval(AT_PLATFORM); + if (platform_str) hasher.Write((const unsigned char*)platform_str, strlen(platform_str) + 1); +# endif +# ifdef AT_EXECFN + const char* exec_str = (const char*)getauxval(AT_EXECFN); + if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); +# endif +#endif // __linux__ + +#ifdef HAVE_GETCPUID + AddAllCPUID(hasher); +#endif + + // Memory locations + hasher << &hasher << &RandAddStaticEnv << &malloc << &errno << &environ; + + // Hostname + char hname[256]; + if (gethostname(hname, 256) == 0) { + hasher.Write((const unsigned char*)hname, strnlen(hname, 256)); + } + +#if HAVE_DECL_GETIFADDRS + // Network interfaces + struct ifaddrs *ifad = NULL; + getifaddrs(&ifad); + struct ifaddrs *ifit = ifad; + while (ifit != NULL) { + hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); + hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); + hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); + AddSockaddr(hasher, ifit->ifa_addr); + AddSockaddr(hasher, ifit->ifa_netmask); + AddSockaddr(hasher, ifit->ifa_dstaddr); + ifit = ifit->ifa_next; + } + freeifaddrs(ifad); +#endif + +#ifndef WIN32 + // UNIX kernel information + struct utsname name; + if (uname(&name) != -1) { + hasher.Write((const unsigned char*)&name.sysname, strlen(name.sysname) + 1); + hasher.Write((const unsigned char*)&name.nodename, strlen(name.nodename) + 1); + hasher.Write((const unsigned char*)&name.release, strlen(name.release) + 1); + hasher.Write((const unsigned char*)&name.version, strlen(name.version) + 1); + hasher.Write((const unsigned char*)&name.machine, strlen(name.machine) + 1); + } + + /* Path and filesystem provided data */ + AddPath(hasher, "/"); + AddPath(hasher, "."); + AddPath(hasher, "/tmp"); + AddPath(hasher, "/home"); + AddPath(hasher, "/proc"); +#ifdef __linux__ + AddFile(hasher, "/proc/cmdline"); + AddFile(hasher, "/proc/cpuinfo"); + AddFile(hasher, "/proc/version"); +#endif + AddFile(hasher, "/etc/passwd"); + AddFile(hasher, "/etc/group"); + AddFile(hasher, "/etc/hosts"); + AddFile(hasher, "/etc/resolv.conf"); + AddFile(hasher, "/etc/timezone"); + AddFile(hasher, "/etc/localtime"); +#endif + + // For MacOS/BSDs, gather data through sysctl instead of /proc. Not all of these + // will exist on every system. +#if HAVE_SYSCTL +# ifdef CTL_HW +# ifdef HW_MACHINE + AddSysctl<CTL_HW, HW_MACHINE>(hasher); +# endif +# ifdef HW_MODEL + AddSysctl<CTL_HW, HW_MODEL>(hasher); +# endif +# ifdef HW_NCPU + AddSysctl<CTL_HW, HW_NCPU>(hasher); +# endif +# ifdef HW_PHYSMEM + AddSysctl<CTL_HW, HW_PHYSMEM>(hasher); +# endif +# ifdef HW_USERMEM + AddSysctl<CTL_HW, HW_USERMEM>(hasher); +# endif +# ifdef HW_MACHINE_ARCH + AddSysctl<CTL_HW, HW_MACHINE_ARCH>(hasher); +# endif +# ifdef HW_REALMEM + AddSysctl<CTL_HW, HW_REALMEM>(hasher); +# endif +# ifdef HW_CPU_FREQ + AddSysctl<CTL_HW, HW_CPU_FREQ>(hasher); +# endif +# ifdef HW_BUS_FREQ + AddSysctl<CTL_HW, HW_BUS_FREQ>(hasher); +# endif +# ifdef HW_CACHELINE + AddSysctl<CTL_HW, HW_CACHELINE>(hasher); +# endif +# endif +# ifdef CTL_KERN +# ifdef KERN_BOOTFILE + AddSysctl<CTL_KERN, KERN_BOOTFILE>(hasher); +# endif +# ifdef KERN_BOOTTIME + AddSysctl<CTL_KERN, KERN_BOOTTIME>(hasher); +# endif +# ifdef KERN_CLOCKRATE + AddSysctl<CTL_KERN, KERN_CLOCKRATE>(hasher); +# endif +# ifdef KERN_HOSTID + AddSysctl<CTL_KERN, KERN_HOSTID>(hasher); +# endif +# ifdef KERN_HOSTUUID + AddSysctl<CTL_KERN, KERN_HOSTUUID>(hasher); +# endif +# ifdef KERN_HOSTNAME + AddSysctl<CTL_KERN, KERN_HOSTNAME>(hasher); +# endif +# ifdef KERN_OSRELDATE + AddSysctl<CTL_KERN, KERN_OSRELDATE>(hasher); +# endif +# ifdef KERN_OSRELEASE + AddSysctl<CTL_KERN, KERN_OSRELEASE>(hasher); +# endif +# ifdef KERN_OSREV + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_OSTYPE + AddSysctl<CTL_KERN, KERN_OSTYPE>(hasher); +# endif +# ifdef KERN_POSIX1 + AddSysctl<CTL_KERN, KERN_OSREV>(hasher); +# endif +# ifdef KERN_VERSION + AddSysctl<CTL_KERN, KERN_VERSION>(hasher); +# endif +# endif +#endif + + // Env variables + if (environ) { + for (size_t i = 0; environ[i]; ++i) { + hasher.Write((const unsigned char*)environ[i], strlen(environ[i])); + } + } + + // Process, thread, user, session, group, ... ids. +#ifdef WIN32 + hasher << GetCurrentProcessId() << GetCurrentThreadId(); +#else + hasher << getpid() << getppid() << getsid(0) << getpgid(0) << getuid() << geteuid() << getgid() << getegid(); +#endif + hasher << std::this_thread::get_id(); +} diff --git a/src/randomenv.h b/src/randomenv.h new file mode 100644 index 0000000000..46cea6f6f2 --- /dev/null +++ b/src/randomenv.h @@ -0,0 +1,17 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RANDOMENV_H +#define BITCOIN_RANDOMENV_H + +#include <crypto/sha512.h> + +/** Gather non-cryptographic environment data that changes over time. */ +void RandAddDynamicEnv(CSHA512& hasher); + +/** Gather non-cryptographic environment data that does not change over time. */ +void RandAddStaticEnv(CSHA512& hasher); + +#endif diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4bbd4aaf64..2f4b4412f5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -10,11 +10,12 @@ #include <chain.h> #include <chainparams.h> #include <coins.h> -#include <node/coinstats.h> #include <consensus/validation.h> #include <core_io.h> #include <hash.h> #include <index/blockfilterindex.h> +#include <node/coinstats.h> +#include <node/utxo_snapshot.h> #include <policy/feerate.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -34,13 +35,10 @@ #include <validationinterface.h> #include <warnings.h> -#include <assert.h> #include <stdint.h> #include <univalue.h> -#include <boost/thread/thread.hpp> // boost::thread::interrupt - #include <condition_variable> #include <memory> #include <mutex> @@ -59,7 +57,7 @@ static CUpdatedBlock latestblock; */ double GetDifficulty(const CBlockIndex* blockindex) { - assert(blockindex); + CHECK_NONFATAL(blockindex); int nShift = (blockindex->nBits >> 24) & 0xff; double dDiff = @@ -958,7 +956,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) PruneBlockFilesManual(height); const CBlockIndex* block = ::ChainActive().Tip(); - assert(block); + CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { block = block->pprev; } @@ -1192,7 +1190,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) {}, RPCResult{ "{\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"chain\": \"xxxx\", (string) current network name (main, test, regtest)\n" " \"blocks\": xxxxxx, (numeric) the height of the most-work fully-validated chain. The genesis block has height 0\n" " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" @@ -1253,7 +1251,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.pushKV("pruned", fPruneMode); if (fPruneMode) { const CBlockIndex* block = tip; - assert(block); + CHECK_NONFATAL(block); while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { block = block->pprev; } @@ -1470,7 +1468,7 @@ static UniValue preciousblock(const JSONRPCRequest& request) } } - CValidationState state; + BlockValidationState state; PreciousBlock(state, Params(), pblockindex); if (!state.IsValid()) { @@ -1495,7 +1493,7 @@ static UniValue invalidateblock(const JSONRPCRequest& request) }.Check(request); uint256 hash(ParseHashV(request.params[0], "blockhash")); - CValidationState state; + BlockValidationState state; CBlockIndex* pblockindex; { @@ -1545,7 +1543,7 @@ static UniValue reconsiderblock(const JSONRPCRequest& request) ResetBlockFailureFlags(pblockindex); } - CValidationState state; + BlockValidationState state; ActivateBestChain(state, Params()); if (!state.IsValid()) { @@ -1599,7 +1597,7 @@ static UniValue getchaintxstats(const JSONRPCRequest& request) } } - assert(pindex != nullptr); + CHECK_NONFATAL(pindex != nullptr); if (request.params[0].isNull()) { blockcount = std::max(0, std::min(blockcount, pindex->nHeight - 1)); @@ -1772,7 +1770,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) } } - assert(pindex != nullptr); + CHECK_NONFATAL(pindex != nullptr); std::set<std::string> stats; if (!request.params[1].isNull()) { @@ -1872,7 +1870,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) } CAmount txfee = tx_total_in - tx_total_out; - assert(MoneyRange(txfee)); + CHECK_NONFATAL(MoneyRange(txfee)); if (do_medianfee) { fee_array.push_back(txfee); } @@ -1976,7 +1974,6 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& Coin coin; if (!cursor->GetKey(key) || !cursor->GetValue(coin)) return false; if (++count % 8192 == 0) { - boost::this_thread::interruption_point(); if (should_abort) { // allow to abort the scan via the abort reference return false; @@ -2009,7 +2006,7 @@ public: explicit CoinsViewScanReserver() : m_could_reserve(false) {} bool reserve() { - assert (!m_could_reserve); + CHECK_NONFATAL(!m_could_reserve); std::lock_guard<std::mutex> lock(g_utxosetscan); if (g_scan_in_progress) { return false; @@ -2136,9 +2133,9 @@ UniValue scantxoutset(const JSONRPCRequest& request) LOCK(cs_main); ::ChainstateActive().ForceFlushStateToDisk(); pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); - assert(pcursor); + CHECK_NONFATAL(pcursor); tip = ::ChainActive().Tip(); - assert(tip); + CHECK_NONFATAL(tip); } bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins); result.pushKV("success", res); @@ -2246,6 +2243,113 @@ static UniValue getblockfilter(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "dumptxoutset", + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest coinsdb (leveldb) to disk.\n", + { + {"path", + RPCArg::Type::STR, + RPCArg::Optional::NO, + /* default_val */ "", + "path to the output file. If relative, will be prefixed by datadir."}, + }, + RPCResult{ + "{\n" + " \"coins_written\": n, (numeric) the number of coins written in the snapshot\n" + " \"base_hash\": \"...\", (string) the hash of the base of the snapshot\n" + " \"base_height\": n, (string) the height of the base of the snapshot\n" + " \"path\": \"...\" (string) the absolute path that the snapshot was written to\n" + "]\n" + }, + RPCExamples{ + HelpExampleCli("dumptxoutset", "utxo.dat") + } + }.Check(request); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + // Write to a temporary path and then move into `path` on completion + // to avoid confusion due to an interruption. + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " already exists. If you are sure this is what you want, " + "move it out of the way first"); + } + + FILE* file{fsbridge::fopen(temppath, "wb")}; + CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + std::unique_ptr<CCoinsViewCursor> pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + // We need to lock cs_main to ensure that the coinsdb isn't written to + // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats + // based upon the coinsdb, and (iii) constructing a cursor to the + // coinsdb for use below this block. + // + // Cursors returned by leveldb iterate over snapshots, so the contents + // of the pcursor will not be affected by simultaneous writes during + // use below this block. + // + // See discussion here: + // https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369 + // + LOCK(::cs_main); + + ::ChainstateActive().ForceFlushStateToDisk(); + + if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); + tip = LookupBlockIndex(stats.hashBlock); + CHECK_NONFATAL(tip); + } + + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; + + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter{0}; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } + + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().ToString()); + result.pushKV("base_height", tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2282,6 +2386,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, + { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on @@ -2290,3 +2395,5 @@ void RegisterBlockchainRPCCommands(CRPCTable &t) for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) t.appendCommand(commands[vcidx].name, &commands[vcidx]); } + +NodeContext* g_rpc_node = nullptr; diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index ff461fbcbc..8a1264f824 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -17,6 +17,7 @@ class CBlock; class CBlockIndex; class CTxMemPool; class UniValue; +struct NodeContext; static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -46,4 +47,9 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); +//! Pointer to node state that needs to be declared as a global to be accessible +//! RPC methods. Due to limitations of the RPC framework, there's currently no +//! direct way to pass in state to RPC methods without globals. +extern NodeContext* g_rpc_node; + #endif diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 32e18312e1..dfca1697c1 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -30,6 +30,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "utxoupdatepsbt", 1, "descriptors" }, { "generatetoaddress", 0, "nblocks" }, { "generatetoaddress", 2, "maxtries" }, + { "generatetodescriptor", 0, "num_blocks" }, + { "generatetodescriptor", 2, "maxtries" }, { "getnetworkhashps", 0, "nblocks" }, { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 07c2958635..ab22155651 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -13,12 +13,15 @@ #include <key_io.h> #include <miner.h> #include <net.h> +#include <node/context.h> #include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/descriptor.h> #include <script/script.h> +#include <script/signingprovider.h> #include <shutdown.h> #include <txmempool.h> #include <univalue.h> @@ -140,6 +143,47 @@ static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, ui return blockHashes; } +static UniValue generatetodescriptor(const JSONRPCRequest& request) +{ + RPCHelpMan{ + "generatetodescriptor", + "\nMine blocks immediately to a specified descriptor (before the RPC call returns)\n", + { + {"num_blocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor to send the newly generated bitcoin to."}, + {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, + }, + RPCResult{ + "[ blockhashes ] (array) hashes of blocks generated\n"}, + RPCExamples{ + "\nGenerate 11 blocks to mydesc\n" + HelpExampleCli("generatetodescriptor", "11 \"mydesc\"")}, + } + .Check(request); + + const int num_blocks{request.params[0].get_int()}; + const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; + + FlatSigningProvider key_provider; + std::string error; + const auto desc = Parse(request.params[1].get_str(), key_provider, error, /* require_checksum = */ false); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + if (desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); + } + + FlatSigningProvider provider; + std::vector<CScript> coinbase_script; + if (!desc->Expand(0, key_provider, coinbase_script, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + CHECK_NONFATAL(coinbase_script.size() == 1); + + return generateBlocks(coinbase_script.at(0), num_blocks, max_tries); +} + static UniValue generatetoaddress(const JSONRPCRequest& request) { RPCHelpMan{"generatetoaddress", @@ -189,7 +233,7 @@ static UniValue getmininginfo(const JSONRPCRequest& request) " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"networkhashps\": nnn, (numeric) The network hashes per second\n" " \"pooledtx\": n (numeric) The size of the mempool\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"chain\": \"xxxx\", (string) current network name (main, test, regtest)\n" " \"warnings\": \"...\" (string) any network and blockchain warnings\n" "}\n" }, @@ -252,7 +296,7 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) // NOTE: Assumes a conclusive result; if result is inconclusive, it must be handled by caller -static UniValue BIP22ValidationResult(const CValidationState& state) +static UniValue BIP22ValidationResult(const BlockValidationState& state) { if (state.IsValid()) return NullUniValue; @@ -401,7 +445,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) // TestBlockValidity only supports blocks built on the current Tip if (block.hashPrevBlock != pindexPrev->GetBlockHash()) return "inconclusive-not-best-prevblk"; - CValidationState state; + BlockValidationState state; TestBlockValidity(state, Params(), block, pindexPrev, false, true); return BIP22ValidationResult(state); } @@ -424,10 +468,10 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) if (strMode != "template") throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - if (g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) + if (g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL) == 0) throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); if (::ChainstateActive().IsInitialBlockDownload()) @@ -511,7 +555,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) // Need to update only after we know CreateNewBlock succeeded pindexPrev = pindexPrevNew; } - assert(pindexPrev); + CHECK_NONFATAL(pindexPrev); CBlock* pblock = &pblocktemplate->block; // pointer for convenience const Consensus::Params& consensusParams = Params().GetConsensus(); @@ -553,7 +597,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) entry.pushKV("fee", pblocktemplate->vTxFees[index_in_template]); int64_t nTxSigOps = pblocktemplate->vTxSigOpsCost[index_in_template]; if (fPreSegWit) { - assert(nTxSigOps % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nTxSigOps % WITNESS_SCALE_FACTOR == 0); nTxSigOps /= WITNESS_SCALE_FACTOR; } entry.pushKV("sigops", nTxSigOps); @@ -642,9 +686,9 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST; int64_t nSizeLimit = MAX_BLOCK_SERIALIZED_SIZE; if (fPreSegWit) { - assert(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); nSigOpLimit /= WITNESS_SCALE_FACTOR; - assert(nSizeLimit % WITNESS_SCALE_FACTOR == 0); + CHECK_NONFATAL(nSizeLimit % WITNESS_SCALE_FACTOR == 0); nSizeLimit /= WITNESS_SCALE_FACTOR; } result.pushKV("sigoplimit", nSigOpLimit); @@ -668,12 +712,12 @@ class submitblock_StateCatcher : public CValidationInterface public: uint256 hash; bool found; - CValidationState state; + BlockValidationState state; explicit submitblock_StateCatcher(const uint256 &hashIn) : hash(hashIn), found(false), state() {} protected: - void BlockChecked(const CBlock& block, const CValidationState& stateIn) override { + void BlockChecked(const CBlock& block, const BlockValidationState& stateIn) override { if (block.GetHash() != hash) return; found = true; @@ -772,8 +816,8 @@ static UniValue submitheader(const JSONRPCRequest& request) } } - CValidationState state; - ProcessNewBlockHeaders({h}, state, Params(), /* ppindex */ nullptr, /* first_invalid */ nullptr); + BlockValidationState state; + ProcessNewBlockHeaders({h}, state, Params()); if (state.IsValid()) return NullUniValue; if (state.IsError()) { throw JSONRPCError(RPC_VERIFY_ERROR, FormatStateMessage(state)); @@ -961,6 +1005,7 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, + { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index d289274a37..d73dd6e52d 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -3,15 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <key_io.h> #include <httpserver.h> +#include <key_io.h> #include <outputtype.h> #include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/util.h> #include <script/descriptor.h> -#include <util/system.h> +#include <util/check.h> #include <util/strencodings.h> +#include <util/system.h> #include <util/validation.h> #include <stdint.h> @@ -540,6 +541,7 @@ static UniValue echo(const JSONRPCRequest& request) throw std::runtime_error( RPCHelpMan{"echo|echojson ...", "\nSimply echo back the input arguments. This command is for testing.\n" + "\nIt will return an internal bug report when exactly 100 arguments are passed.\n" "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in " "bitcoin-cli and the GUI. There is no server-side difference.", {}, @@ -548,6 +550,8 @@ static UniValue echo(const JSONRPCRequest& request) }.ToString() ); + CHECK_NONFATAL(request.params.size() != 100); + return request.params; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 7b1507e4dc..f1dcc9b607 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -8,10 +8,13 @@ #include <clientversion.h> #include <core_io.h> #include <net.h> -#include <net_processing.h> #include <net_permissions.h> +#include <net_processing.h> +#include <net_types.h> // For banmap_t #include <netbase.h> +#include <node/context.h> #include <policy/settings.h> +#include <rpc/blockchain.h> #include <rpc/protocol.h> #include <rpc/util.h> #include <sync.h> @@ -38,10 +41,10 @@ static UniValue getconnectioncount(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - return (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL); + return (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL); } static UniValue ping(const JSONRPCRequest& request) @@ -58,11 +61,11 @@ static UniValue ping(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); // Request that each node send a ping during next message processing pass - g_connman->ForEachNode([](CNode* pnode) { + g_rpc_node->connman->ForEachNode([](CNode* pnode) { pnode->fPingQueued = true; }); return NullUniValue; @@ -131,11 +134,11 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::vector<CNodeStats> vstats; - g_connman->GetNodeStats(vstats); + g_rpc_node->connman->GetNodeStats(vstats); UniValue ret(UniValue::VARR); @@ -234,7 +237,7 @@ static UniValue addnode(const JSONRPCRequest& request) }, }.ToString()); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); std::string strNode = request.params[0].get_str(); @@ -242,18 +245,18 @@ static UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); + g_rpc_node->connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } if (strCommand == "add") { - if(!g_connman->AddNode(strNode)) + if(!g_rpc_node->connman->AddNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: Node already added"); } else if(strCommand == "remove") { - if(!g_connman->RemoveAddedNode(strNode)) + if(!g_rpc_node->connman->RemoveAddedNode(strNode)) throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); } @@ -279,7 +282,7 @@ static UniValue disconnectnode(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); bool success; @@ -288,11 +291,11 @@ static UniValue disconnectnode(const JSONRPCRequest& request) if (!address_arg.isNull() && id_arg.isNull()) { /* handle disconnect-by-address */ - success = g_connman->DisconnectNode(address_arg.get_str()); + success = g_rpc_node->connman->DisconnectNode(address_arg.get_str()); } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { /* handle disconnect-by-id */ NodeId nodeid = (NodeId) id_arg.get_int64(); - success = g_connman->DisconnectNode(nodeid); + success = g_rpc_node->connman->DisconnectNode(nodeid); } else { throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); } @@ -333,10 +336,10 @@ static UniValue getaddednodeinfo(const JSONRPCRequest& request) }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - std::vector<AddedNodeInfo> vInfo = g_connman->GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = g_rpc_node->connman->GetAddedNodeInfo(); if (!request.params[0].isNull()) { bool found = false; @@ -399,21 +402,21 @@ static UniValue getnettotals(const JSONRPCRequest& request) + HelpExampleRpc("getnettotals", "") }, }.Check(request); - if(!g_connman) + if(!g_rpc_node->connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); UniValue obj(UniValue::VOBJ); - obj.pushKV("totalbytesrecv", g_connman->GetTotalBytesRecv()); - obj.pushKV("totalbytessent", g_connman->GetTotalBytesSent()); + obj.pushKV("totalbytesrecv", g_rpc_node->connman->GetTotalBytesRecv()); + obj.pushKV("totalbytessent", g_rpc_node->connman->GetTotalBytesSent()); obj.pushKV("timemillis", GetTimeMillis()); UniValue outboundLimit(UniValue::VOBJ); - outboundLimit.pushKV("timeframe", g_connman->GetMaxOutboundTimeframe()); - outboundLimit.pushKV("target", g_connman->GetMaxOutboundTarget()); - outboundLimit.pushKV("target_reached", g_connman->OutboundTargetReached(false)); - outboundLimit.pushKV("serve_historical_blocks", !g_connman->OutboundTargetReached(true)); - outboundLimit.pushKV("bytes_left_in_cycle", g_connman->GetOutboundTargetBytesLeft()); - outboundLimit.pushKV("time_left_in_cycle", g_connman->GetMaxOutboundTimeLeftInCycle()); + outboundLimit.pushKV("timeframe", g_rpc_node->connman->GetMaxOutboundTimeframe()); + outboundLimit.pushKV("target", g_rpc_node->connman->GetMaxOutboundTarget()); + outboundLimit.pushKV("target_reached", g_rpc_node->connman->OutboundTargetReached(false)); + outboundLimit.pushKV("serve_historical_blocks", !g_rpc_node->connman->OutboundTargetReached(true)); + outboundLimit.pushKV("bytes_left_in_cycle", g_rpc_node->connman->GetOutboundTargetBytesLeft()); + outboundLimit.pushKV("time_left_in_cycle", g_rpc_node->connman->GetMaxOutboundTimeLeftInCycle()); obj.pushKV("uploadtarget", outboundLimit); return obj; } @@ -492,16 +495,16 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) obj.pushKV("version", CLIENT_VERSION); obj.pushKV("subversion", strSubVersion); obj.pushKV("protocolversion",PROTOCOL_VERSION); - if (g_connman) { - ServiceFlags services = g_connman->GetLocalServices(); + if (g_rpc_node->connman) { + ServiceFlags services = g_rpc_node->connman->GetLocalServices(); obj.pushKV("localservices", strprintf("%016x", services)); obj.pushKV("localservicesnames", GetServicesNames(services)); } obj.pushKV("localrelay", g_relay_txes); obj.pushKV("timeoffset", GetTimeOffset()); - if (g_connman) { - obj.pushKV("networkactive", g_connman->GetNetworkActive()); - obj.pushKV("connections", (int)g_connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); + if (g_rpc_node->connman) { + obj.pushKV("networkactive", g_rpc_node->connman->GetNetworkActive()); + obj.pushKV("connections", (int)g_rpc_node->connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); } obj.pushKV("networks", GetNetworksInfo()); obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -546,7 +549,7 @@ static UniValue setban(const JSONRPCRequest& request) if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { throw std::runtime_error(help.ToString()); } - if (!g_banman) { + if (!g_rpc_node->banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } @@ -570,7 +573,7 @@ static UniValue setban(const JSONRPCRequest& request) if (strCommand == "add") { - if (isSubnet ? g_banman->IsBanned(subNet) : g_banman->IsBanned(netAddr)) { + if (isSubnet ? g_rpc_node->banman->IsBanned(subNet) : g_rpc_node->banman->IsBanned(netAddr)) { throw JSONRPCError(RPC_CLIENT_NODE_ALREADY_ADDED, "Error: IP/Subnet already banned"); } @@ -583,20 +586,20 @@ static UniValue setban(const JSONRPCRequest& request) absolute = true; if (isSubnet) { - g_banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); - if (g_connman) { - g_connman->DisconnectNode(subNet); + g_rpc_node->banman->Ban(subNet, BanReasonManuallyAdded, banTime, absolute); + if (g_rpc_node->connman) { + g_rpc_node->connman->DisconnectNode(subNet); } } else { - g_banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); - if (g_connman) { - g_connman->DisconnectNode(netAddr); + g_rpc_node->banman->Ban(netAddr, BanReasonManuallyAdded, banTime, absolute); + if (g_rpc_node->connman) { + g_rpc_node->connman->DisconnectNode(netAddr); } } } else if(strCommand == "remove") { - if (!( isSubnet ? g_banman->Unban(subNet) : g_banman->Unban(netAddr) )) { + if (!( isSubnet ? g_rpc_node->banman->Unban(subNet) : g_rpc_node->banman->Unban(netAddr) )) { throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Unban failed. Requested address/subnet was not previously banned."); } } @@ -615,12 +618,12 @@ static UniValue listbanned(const JSONRPCRequest& request) }, }.Check(request); - if(!g_banman) { + if(!g_rpc_node->banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } banmap_t banMap; - g_banman->GetBanned(banMap); + g_rpc_node->banman->GetBanned(banMap); UniValue bannedAddresses(UniValue::VARR); for (const auto& entry : banMap) @@ -649,11 +652,11 @@ static UniValue clearbanned(const JSONRPCRequest& request) + HelpExampleRpc("clearbanned", "") }, }.Check(request); - if (!g_banman) { + if (!g_rpc_node->banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } - g_banman->ClearBanned(); + g_rpc_node->banman->ClearBanned(); return NullUniValue; } @@ -669,13 +672,13 @@ static UniValue setnetworkactive(const JSONRPCRequest& request) RPCExamples{""}, }.Check(request); - if (!g_connman) { + if (!g_rpc_node->connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } - g_connman->SetNetworkActive(request.params[0].get_bool()); + g_rpc_node->connman->SetNetworkActive(request.params[0].get_bool()); - return g_connman->GetNetworkActive(); + return g_rpc_node->connman->GetNetworkActive(); } static UniValue getnodeaddresses(const JSONRPCRequest& request) @@ -701,7 +704,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) + HelpExampleRpc("getnodeaddresses", "8") }, }.Check(request); - if (!g_connman) { + if (!g_rpc_node->connman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -713,7 +716,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) } } // returns a shuffled list of CAddress - std::vector<CAddress> vAddr = g_connman->GetAddresses(); + std::vector<CAddress> vAddr = g_rpc_node->connman->GetAddresses(); UniValue ret(UniValue::VARR); int address_return_count = std::min<int>(count, vAddr.size()); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index a16e5cb61a..983f251d6b 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -11,6 +11,7 @@ #include <key_io.h> #include <merkleblock.h> #include <node/coin.h> +#include <node/context.h> #include <node/psbt.h> #include <node/transaction.h> #include <policy/policy.h> @@ -18,6 +19,7 @@ #include <primitives/transaction.h> #include <psbt.h> #include <random.h> +#include <rpc/blockchain.h> #include <rpc/rawtransaction_util.h> #include <rpc/server.h> #include <rpc/util.h> @@ -817,7 +819,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) std::string err_string; AssertLockNotHeld(cs_main); - const TransactionError err = BroadcastTransaction(tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); + const TransactionError err = BroadcastTransaction(*g_rpc_node, tx, err_string, max_raw_tx_fee, /*relay*/ true, /*wait_callback*/ true); if (TransactionError::OK != err) { throw JSONRPCTransactionError(err, err_string); } @@ -893,20 +895,21 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) UniValue result_0(UniValue::VOBJ); result_0.pushKV("txid", tx_hash.GetHex()); - CValidationState state; - bool missing_inputs; + TxValidationState state; bool test_accept_res; { LOCK(cs_main); - test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), &missing_inputs, + test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), nullptr /* plTxnReplaced */, false /* bypass_limits */, max_raw_tx_fee, /* test_accept */ true); } result_0.pushKV("allowed", test_accept_res); if (!test_accept_res) { if (state.IsInvalid()) { - result_0.pushKV("reject-reason", strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason())); - } else if (missing_inputs) { - result_0.pushKV("reject-reason", "missing-inputs"); + if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { + result_0.pushKV("reject-reason", "missing-inputs"); + } else { + result_0.pushKV("reject-reason", strprintf("%s", state.GetRejectReason())); + } } else { result_0.pushKV("reject-reason", state.GetRejectReason()); } @@ -1602,7 +1605,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) for (auto& psbt : psbtxs) { for (unsigned int i = 0; i < psbt.tx->vin.size(); ++i) { if (!merged_psbt.AddInput(psbt.tx->vin[i], psbt.inputs[i])) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input %s:%d exists in multiple PSBTs", psbt.tx->vin[i].prevout.hash.ToString().c_str(), psbt.tx->vin[i].prevout.n)); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input %s:%d exists in multiple PSBTs", psbt.tx->vin[i].prevout.hash.ToString(), psbt.tx->vin[i].prevout.n)); } } for (unsigned int i = 0; i < psbt.tx->vout.size(); ++i) { @@ -1617,7 +1620,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) std::vector<int> output_indices(merged_psbt.outputs.size()); std::iota(output_indices.begin(), output_indices.end(), 0); - // Shuffle input and output indicies lists + // Shuffle input and output indices lists Shuffle(input_indices.begin(), input_indices.end(), FastRandomContext()); Shuffle(output_indices.begin(), output_indices.end(), FastRandomContext()); diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 5b92650764..1936998ff3 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -29,7 +29,7 @@ UniValue SignTransaction(CMutableTransaction& mtx, const SigningProvider* keysto * Parse a prevtxs UniValue array and get the map of coins from it * * @param prevTxs Array of previous txns outputs that tx depends on but may not yet be in the block chain - * @param keystore A pointer to the temprorary keystore if there is one + * @param keystore A pointer to the temporary keystore if there is one * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call */ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keystore, std::map<COutPoint, Coin>& coins); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index adda90c104..cfa3509c65 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -13,8 +13,6 @@ #include <tuple> -InitInterfaces* g_rpc_interfaces = nullptr; - void RPCTypeCheck(const UniValue& params, const std::list<UniValueType>& typesExpected, bool fAllowNull) @@ -430,7 +428,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP std::set<std::string> named_args; for (const auto& arg : m_args) { // Should have unique named arguments - assert(named_args.insert(arg.m_name).second); + CHECK_NONFATAL(named_args.insert(arg.m_name).second); } } @@ -622,11 +620,11 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code - assert(false); + CHECK_NONFATAL(false); // no default case, so the compiler can warn about missing cases } - assert(false); + CHECK_NONFATAL(false); } std::string RPCArg::ToString(const bool oneline) const @@ -663,7 +661,7 @@ std::string RPCArg::ToString(const bool oneline) const // no default case, so the compiler can warn about missing cases } - assert(false); + CHECK_NONFATAL(false); } static std::pair<int64_t, int64_t> ParseRange(const UniValue& value) diff --git a/src/rpc/util.h b/src/rpc/util.h index 72fc7b6286..221638aa9e 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -7,14 +7,15 @@ #include <node/transaction.h> #include <outputtype.h> -#include <pubkey.h> #include <protocol.h> +#include <pubkey.h> #include <rpc/protocol.h> #include <rpc/request.h> #include <script/script.h> #include <script/sign.h> #include <script/standard.h> #include <univalue.h> +#include <util/check.h> #include <string> #include <vector> @@ -24,12 +25,6 @@ class FillableSigningProvider; class CPubKey; class CScript; -struct InitInterfaces; - -//! Pointers to interfaces that need to be accessible from RPC methods. Due to -//! limitations of the RPC framework, there's currently no direct way to pass in -//! state to RPC method implementations. -extern InitInterfaces* g_rpc_interfaces; /** Wrapper for UniValue::VType, which includes typeAny: * Used to denote don't care type. */ @@ -146,7 +141,7 @@ struct RPCArg { m_oneline_description{oneline_description}, m_type_str{type_str} { - assert(type != Type::ARR && type != Type::OBJ); + CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ); } RPCArg( @@ -165,7 +160,7 @@ struct RPCArg { m_oneline_description{oneline_description}, m_type_str{type_str} { - assert(type == Type::ARR || type == Type::OBJ); + CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ); } bool IsOptional() const; @@ -194,14 +189,14 @@ struct RPCResult { explicit RPCResult(std::string result) : m_cond{}, m_result{std::move(result)} { - assert(!m_result.empty()); + CHECK_NONFATAL(!m_result.empty()); } RPCResult(std::string cond, std::string result) : m_cond{std::move(cond)}, m_result{std::move(result)} { - assert(!m_cond.empty()); - assert(!m_result.empty()); + CHECK_NONFATAL(!m_cond.empty()); + CHECK_NONFATAL(!m_result.empty()); } }; diff --git a/src/scheduler.cpp b/src/scheduler.cpp index fdc859b3a0..07a54335ac 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -41,8 +41,6 @@ void CScheduler::serviceQueue() try { if (!shouldStop() && taskQueue.empty()) { reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); - // Use this chance to get more entropy - RandAddSeedSleep(); } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 536807e1d8..13cdd6c61a 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -36,7 +36,7 @@ namespace { // xpubs use other characters too, but already have their own checksum // mechanism. // * Function names like "multi()" use other characters, but mistakes in -// these would generally result in an unparseable descriptor. +// these would generally result in an unparsable descriptor. // * A case error always counts as 1 symbol error. // * Any other 1 character substitution error counts as 1 or 2 symbol errors. // * Any 1 symbol error is always detected. @@ -645,7 +645,7 @@ NODISCARD bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& } uint32_t p; if (!ParseUInt32(std::string(elem.begin(), elem.end()), &p)) { - error = strprintf("Key path value '%s' is not a valid uint32", std::string(elem.begin(), elem.end()).c_str()); + error = strprintf("Key path value '%s' is not a valid uint32", std::string(elem.begin(), elem.end())); return false; } else if (p > 0x7FFFFFFFUL) { error = strprintf("Key path value %u is out of range", p); @@ -783,7 +783,7 @@ std::unique_ptr<DescriptorImpl> ParseScript(Span<const char>& sp, ParseScriptCon uint32_t thres; std::vector<std::unique_ptr<PubkeyProvider>> providers; if (!ParseUInt32(std::string(threshold.begin(), threshold.end()), &thres)) { - error = strprintf("Multi threshold '%s' is not valid", std::string(threshold.begin(), threshold.end()).c_str()); + error = strprintf("Multi threshold '%s' is not valid", std::string(threshold.begin(), threshold.end())); return nullptr; } size_t script_size = 0; diff --git a/src/script/signingprovider.h b/src/script/signingprovider.h index 4eec2311d4..c40fecac5c 100644 --- a/src/script/signingprovider.h +++ b/src/script/signingprovider.h @@ -63,8 +63,6 @@ FlatSigningProvider Merge(const FlatSigningProvider& a, const FlatSigningProvide class FillableSigningProvider : public SigningProvider { protected: - mutable CCriticalSection cs_KeyStore; - using KeyMap = std::map<CKeyID, CKey>; using ScriptMap = std::map<CScriptID, CScript>; @@ -74,6 +72,8 @@ protected: void ImplicitlyLearnRelatedKeyScripts(const CPubKey& pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); public: + mutable CCriticalSection cs_KeyStore; + virtual bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey); virtual bool AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } virtual bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; diff --git a/src/sync.cpp b/src/sync.cpp index 653800ae4e..257093fad1 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -173,7 +173,7 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, for (const std::pair<void*, CLockLocation>& i : g_lockstack) if (i.first == cs) return; - tfm::format(std::cerr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + tfm::format(std::cerr, "Assertion failed: lock %s not held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } @@ -181,7 +181,7 @@ void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLi { for (const std::pair<void*, CLockLocation>& i : g_lockstack) { if (i.first == cs) { - tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } } diff --git a/src/test/README.md b/src/test/README.md index 96dcb072bc..731720f654 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -7,8 +7,8 @@ configure some other framework (we want as few impediments to creating unit tests as possible). The build system is set up to compile an executable called `test_bitcoin` -that runs all of the unit tests. The main source file is called -`setup_common.cpp`. +that runs all of the unit tests. The main source file for the test library is found in +`util/setup_common.cpp`. ### Compiling/running unit tests diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index da0abd495a..c034216bc1 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index e333763f27..d33d668a04 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -5,7 +5,7 @@ #include <util/memory.h> #include <util/system.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <memory> diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 378fe285d5..e20900ed13 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -4,7 +4,7 @@ #include <amount.h> #include <policy/feerate.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 9ac87261b6..3723a48903 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -11,7 +11,7 @@ #include <uint256.h> #include <arith_uint256.h> #include <string> -#include <test/setup_common.h> +#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index b3bed2434c..bd6ece935b 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index cb376cddb6..52301f799a 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -5,7 +5,7 @@ #include <test/data/base58_encode_decode.json.h> #include <base58.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <univalue.h> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 9ffffb0b7d..a5fed55504 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index 0ba492c24e..a2098f4f56 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -3,25 +3,13 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bech32.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> +#include <test/util/str.h> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(bech32_tests, BasicTestingSetup) -static bool CaseInsensitiveEqual(const std::string &s1, const std::string &s2) -{ - if (s1.size() != s2.size()) return false; - for (size_t i = 0; i < s1.size(); ++i) { - char c1 = s1[i]; - if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); - char c2 = s2[i]; - if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); - if (c1 != c2) return false; - } - return true; -} - BOOST_AUTO_TEST_CASE(bip173_testvectors_valid) { static const std::string CASES[] = { diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index 660df00964..53df032252 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -9,7 +9,7 @@ #include <key_io.h> #include <streams.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <vector> @@ -118,22 +118,6 @@ static void RunTest(const TestVector &test) { } key = keyNew; pubkey = pubkeyNew; - - CDataStream ssPub(SER_DISK, CLIENT_VERSION); - ssPub << pubkeyNew; - BOOST_CHECK(ssPub.size() == 75); - - CDataStream ssPriv(SER_DISK, CLIENT_VERSION); - ssPriv << keyNew; - BOOST_CHECK(ssPriv.size() == 75); - - CExtPubKey pubCheck; - CExtKey privCheck; - ssPub >> pubCheck; - ssPriv >> privCheck; - - BOOST_CHECK(pubCheck == pubkeyNew); - BOOST_CHECK(privCheck == keyNew); } } diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index ca75563ef0..29935c09f8 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -4,7 +4,7 @@ #include <chain.h> #include <rpc/blockchain.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 5ce8e6feb0..8694891a51 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -3,22 +3,18 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockencodings.h> -#include <consensus/merkle.h> #include <chainparams.h> +#include <consensus/merkle.h> #include <pow.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> std::vector<std::pair<uint256, CTransactionRef>> extra_txn; -struct RegtestingSetup : public TestingSetup { - RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} -}; - -BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegtestingSetup) +BOOST_FIXTURE_TEST_SUITE(blockencodings_tests, RegTestingSetup) static CBlock BuildBlockTestCase() { CBlock block; diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index ba293b7836..2b616f4c2b 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -8,8 +8,9 @@ #include <index/blockfilterindex.h> #include <miner.h> #include <pow.h> -#include <test/setup_common.h> #include <script/standard.h> +#include <test/util/blockfilter.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <validation.h> @@ -17,23 +18,6 @@ BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) -static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, - BlockFilter& filter) -{ - CBlock block; - if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { - return false; - } - - CBlockUndo block_undo; - if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { - return false; - } - - filter = BlockFilter(filter_type, block, block_undo); - return true; -} - static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, uint256& last_header) { @@ -102,8 +86,8 @@ static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key)); CBlockHeader header = block->GetBlockHeader(); - CValidationState state; - if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex, nullptr)) { + BlockValidationState state; + if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex)) { return false; } } diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index df0a041e0e..e69503ef35 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/data/blockfilters.json.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <blockfilter.h> #include <core_io.h> diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 4421494007..4a7ad9b38b 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -15,7 +15,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -461,7 +461,7 @@ static std::vector<unsigned char> RandomData() BOOST_AUTO_TEST_CASE(rolling_bloom) { - SeedInsecureRand(/* deterministic */ true); + SeedInsecureRand(SeedRand::ZEROS); g_mock_deterministic_tests = true; // last-100-entry, 1% false positive: diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 8fd4e5d5d6..d5e2344a8b 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compat/byteswap.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index d796444419..6745bb9015 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -5,9 +5,8 @@ #include <util/memory.h> #include <util/system.h> #include <util/time.h> -#include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> @@ -19,11 +18,10 @@ #include <unordered_set> -// BasicTestingSetup not sufficient because nScriptCheckThreads is not set -// otherwise. BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) static const unsigned int QUEUE_BATCH_SIZE = 128; +static const int SCRIPT_CHECK_THREADS = 3; struct FakeCheck { bool operator()() @@ -149,7 +147,7 @@ static void Correct_Queue_range(std::vector<size_t> range) { auto small_queue = MakeUnique<Correct_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{small_queue->Thread();}); } // Make vChecks here to save on malloc (this test can be slow...) @@ -214,7 +212,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{fail_queue->Thread();}); } @@ -246,7 +244,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{fail_queue->Thread();}); } @@ -274,7 +272,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) { auto queue = MakeUnique<Unique_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{queue->Thread();}); } @@ -310,7 +308,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) { auto queue = MakeUnique<Memory_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{queue->Thread();}); } for (size_t i = 0; i < 1000; ++i) { @@ -342,7 +340,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) auto queue = MakeUnique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE); boost::thread_group tg; bool fails = false; - for (auto x = 0; x < nScriptCheckThreads; ++x) { + for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { tg.create_thread([&]{queue->Thread();}); } std::thread t0([&]() { diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 948591196c..436c1bffa0 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -7,7 +7,7 @@ #include <coins.h> #include <script/standard.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <uint256.h> #include <undo.h> #include <util/strencodings.h> @@ -279,7 +279,7 @@ UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { // has the expected effect (the other duplicate is overwritten at all cache levels) BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { - SeedInsecureRand(/* deterministic */ true); + SeedInsecureRand(SeedRand::ZEROS); g_mock_deterministic_tests = true; bool spent_a_duplicate_coinbase = false; diff --git a/src/test/compilerbug_tests.cpp b/src/test/compilerbug_tests.cpp index 74e1eac3ea..1a6fcda009 100644 --- a/src/test/compilerbug_tests.cpp +++ b/src/test/compilerbug_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(compilerbug_tests, BasicTestingSetup) diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index e8f149470e..22eae91cf0 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -3,7 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> +#include <script/standard.h> #include <stdint.h> @@ -61,4 +62,76 @@ BOOST_AUTO_TEST_CASE(compress_amounts) BOOST_CHECK(TestDecode(i)); } +BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) +{ + // case CKeyID + CKey key; + key.MakeNewKey(true); + CPubKey pubkey = key.GetPubKey(); + + CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; + BOOST_CHECK_EQUAL(script.size(), 25); + + std::vector<unsigned char> out; + bool done = CompressScript(script, out); + BOOST_CHECK_EQUAL(done, true); + + // Check compressed script + BOOST_CHECK_EQUAL(out.size(), 21); + BOOST_CHECK_EQUAL(out[0], 0x00); + BOOST_CHECK_EQUAL(memcmp(&out[1], &script[3], 20), 0); // compare the 20 relevant chars of the CKeyId in the script +} + +BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) +{ + // case CScriptID + CScript script, redeemScript; + script << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; + BOOST_CHECK_EQUAL(script.size(), 23); + + std::vector<unsigned char> out; + bool done = CompressScript(script, out); + BOOST_CHECK_EQUAL(done, true); + + // Check compressed script + BOOST_CHECK_EQUAL(out.size(), 21); + BOOST_CHECK_EQUAL(out[0], 0x01); + BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 20), 0); // compare the 20 relevant chars of the CScriptId in the script +} + +BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) +{ + CKey key; + key.MakeNewKey(true); // case compressed PubKeyID + + CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) + BOOST_CHECK_EQUAL(script.size(), 35); + + std::vector<unsigned char> out; + bool done = CompressScript(script, out); + BOOST_CHECK_EQUAL(done, true); + + // Check compressed script + BOOST_CHECK_EQUAL(out.size(), 33); + BOOST_CHECK_EQUAL(memcmp(&out[0], &script[1], 1), 0); + BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // compare the 32 chars of the compressed CPubKey +} + +BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) +{ + CKey key; + key.MakeNewKey(false); // case uncompressed PubKeyID + CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) + BOOST_CHECK_EQUAL(script.size(), 67); // 1 char code + 65 char pubkey + OP_CHECKSIG + + std::vector<unsigned char> out; + bool done = CompressScript(script, out); + BOOST_CHECK_EQUAL(done, true); + + // Check compressed script + BOOST_CHECK_EQUAL(out.size(), 33); + BOOST_CHECK_EQUAL(memcmp(&out[1], &script[2], 32), 0); // first 32 chars of CPubKey are copied into out[1:] + BOOST_CHECK_EQUAL(out[0], 0x04 | (script[65] & 0x01)); // least significant bit (lsb) of last char of pubkey is mapped into out[0] +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4ac12bf969..591a317d17 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -15,7 +15,7 @@ #include <crypto/sha512.h> #include <random.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index d38ede691a..119d4b3295 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -4,17 +4,17 @@ #include <boost/test/unit_test.hpp> #include <cuckoocache.h> #include <script/sigcache.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <random.h> #include <thread> /** Test Suite for CuckooCache * - * 1) All tests should have a deterministic result (using insecure rand + * 1. All tests should have a deterministic result (using insecure rand * with deterministic seeds) - * 2) Some test methods are templated to allow for easier testing + * 2. Some test methods are templated to allow for easier testing * against new versions / comparing - * 3) Results should be treated as a regression test, i.e., did the behavior + * 3. Results should be treated as a regression test, i.e., did the behavior * change significantly from what was expected. This can be OK, depending on * the nature of the change, but requires updating the tests to reflect the new * expected behavior. For example improving the hit rate may cause some tests @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_SUITE(cuckoocache_tests); */ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) template <typename Cache> static double test_cache(size_t megabytes, double load) { - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -82,9 +82,9 @@ static double test_cache(size_t megabytes, double load) * * Examples: * - * 1) at load 0.5, we expect a perfect hit rate, so we multiply by + * 1. at load 0.5, we expect a perfect hit rate, so we multiply by * 1.0 - * 2) at load 2.0, we expect to see half the entries, so a perfect hit rate + * 2. at load 2.0, we expect to see half the entries, so a perfect hit rate * would be 0.5. Therefore, if we see a hit rate of 0.4, 0.4*2.0 = 0.8 is the * normalized hit rate. * @@ -118,7 +118,7 @@ template <typename Cache> static void test_cache_erase(size_t megabytes) { double load = 1; - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -181,7 +181,7 @@ template <typename Cache> static void test_cache_erase_parallel(size_t megabytes) { double load = 1; - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -285,7 +285,7 @@ static void test_cache_generations() // iterations with non-deterministic values, so it isn't "overfit" to the // specific entropy in FastRandomContext(true) and implementation of the // cache. - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); // block_activity models a chunk of network activity. n_insert elements are // added to the cache. The first and last n/4 are stored for removal later diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index efcadd51fc..57d5b2bb5c 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -4,7 +4,7 @@ #include <dbwrapper.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/memory.h> #include <memory> @@ -42,6 +42,86 @@ BOOST_AUTO_TEST_CASE(dbwrapper) } } +BOOST_AUTO_TEST_CASE(dbwrapper_basic_data) +{ + // Perform tests both obfuscated and non-obfuscated. + for (bool obfuscate : {false, true}) { + fs::path ph = GetDataDir() / (obfuscate ? "dbwrapper_1_obfuscate_true" : "dbwrapper_1_obfuscate_false"); + CDBWrapper dbw(ph, (1 << 20), false, true, obfuscate); + + uint256 res; + uint32_t res_uint_32; + bool res_bool; + + // Ensure that we're doing real obfuscation when obfuscate=true + BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw))); + + //Simulate block raw data - "b + block hash" + std::string key_block = "b" + InsecureRand256().ToString(); + + uint256 in_block = InsecureRand256(); + BOOST_CHECK(dbw.Write(key_block, in_block)); + BOOST_CHECK(dbw.Read(key_block, res)); + BOOST_CHECK_EQUAL(res.ToString(), in_block.ToString()); + + //Simulate file raw data - "f + file_number" + std::string key_file = strprintf("f%04x", InsecureRand32()); + + uint256 in_file_info = InsecureRand256(); + BOOST_CHECK(dbw.Write(key_file, in_file_info)); + BOOST_CHECK(dbw.Read(key_file, res)); + BOOST_CHECK_EQUAL(res.ToString(), in_file_info.ToString()); + + //Simulate transaction raw data - "t + transaction hash" + std::string key_transaction = "t" + InsecureRand256().ToString(); + + uint256 in_transaction = InsecureRand256(); + BOOST_CHECK(dbw.Write(key_transaction, in_transaction)); + BOOST_CHECK(dbw.Read(key_transaction, res)); + BOOST_CHECK_EQUAL(res.ToString(), in_transaction.ToString()); + + //Simulate UTXO raw data - "c + transaction hash" + std::string key_utxo = "c" + InsecureRand256().ToString(); + + uint256 in_utxo = InsecureRand256(); + BOOST_CHECK(dbw.Write(key_utxo, in_utxo)); + BOOST_CHECK(dbw.Read(key_utxo, res)); + BOOST_CHECK_EQUAL(res.ToString(), in_utxo.ToString()); + + //Simulate last block file number - "l" + char key_last_blockfile_number = 'l'; + uint32_t lastblockfilenumber = InsecureRand32(); + BOOST_CHECK(dbw.Write(key_last_blockfile_number, lastblockfilenumber)); + BOOST_CHECK(dbw.Read(key_last_blockfile_number, res_uint_32)); + BOOST_CHECK_EQUAL(lastblockfilenumber, res_uint_32); + + //Simulate Is Reindexing - "R" + char key_IsReindexing = 'R'; + bool isInReindexing = InsecureRandBool(); + BOOST_CHECK(dbw.Write(key_IsReindexing, isInReindexing)); + BOOST_CHECK(dbw.Read(key_IsReindexing, res_bool)); + BOOST_CHECK_EQUAL(isInReindexing, res_bool); + + //Simulate last block hash up to which UXTO covers - 'B' + char key_lastblockhash_uxto = 'B'; + uint256 lastblock_hash = InsecureRand256(); + BOOST_CHECK(dbw.Write(key_lastblockhash_uxto, lastblock_hash)); + BOOST_CHECK(dbw.Read(key_lastblockhash_uxto, res)); + BOOST_CHECK_EQUAL(lastblock_hash, res); + + //Simulate file raw data - "F + filename_number + filename" + std::string file_option_tag = "F"; + uint8_t filename_length = InsecureRandBits(8); + std::string filename = "randomfilename"; + std::string key_file_option = strprintf("%s%01x%s", file_option_tag,filename_length,filename); + + bool in_file_bool = InsecureRandBool(); + BOOST_CHECK(dbw.Write(key_file_option, in_file_bool)); + BOOST_CHECK(dbw.Read(key_file_option, res_bool)); + BOOST_CHECK_EQUAL(res_bool, in_file_bool); + } +} + // Test batch operations BOOST_AUTO_TEST_CASE(dbwrapper_batch) { diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 1928324b27..632151793e 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -17,7 +17,7 @@ #include <util/time.h> #include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <stdint.h> diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 55726a4a8f..bcce8854e3 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -6,7 +6,7 @@ #include <string> #include <script/sign.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <script/descriptor.h> #include <util/strencodings.h> diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index 740d805cce..9bb0b3ef02 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -5,7 +5,7 @@ #include <clientversion.h> #include <flatfile.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index b504a3cbb1..d02c3613ba 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <fs.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/fuzz/bech32.cpp b/src/test/fuzz/bech32.cpp new file mode 100644 index 0000000000..8b91f9bc96 --- /dev/null +++ b/src/test/fuzz/bech32.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bech32.h> +#include <test/fuzz/fuzz.h> +#include <test/util/str.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <utility> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string random_string(buffer.begin(), buffer.end()); + const std::pair<std::string, std::vector<uint8_t>> r1 = bech32::Decode(random_string); + if (r1.first.empty()) { + assert(r1.second.empty()); + } else { + const std::string& hrp = r1.first; + const std::vector<uint8_t>& data = r1.second; + const std::string reencoded = bech32::Encode(hrp, data); + assert(CaseInsensitiveEqual(random_string, reencoded)); + } + + std::vector<unsigned char> input; + ConvertBits<8, 5, true>([&](unsigned char c) { input.push_back(c); }, buffer.begin(), buffer.end()); + const std::string encoded = bech32::Encode("bc", input); + assert(!encoded.empty()); + + const std::pair<std::string, std::vector<uint8_t>> r2 = bech32::Decode(encoded); + if (r2.first.empty()) { + assert(r2.second.empty()); + } else { + const std::string& hrp = r2.first; + const std::vector<uint8_t>& data = r2.second; + assert(hrp == "bc"); + assert(data == input); + } +} diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp new file mode 100644 index 0000000000..c4c25854fd --- /dev/null +++ b/src/test/fuzz/descriptor_parse.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <script/descriptor.h> +#include <test/fuzz/fuzz.h> + +void initialize() +{ + SelectParams(CBaseChainParams::REGTEST); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string descriptor(buffer.begin(), buffer.end()); + FlatSigningProvider signing_provider; + std::string error; + for (const bool require_checksum : {true, false}) { + Parse(descriptor, signing_provider, error, require_checksum); + } +} diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 3a6876ad39..bcd8691359 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -12,6 +12,7 @@ #include <net.h> #include <primitives/block.h> #include <protocol.h> +#include <pubkey.h> #include <streams.h> #include <undo.h> #include <version.h> @@ -23,6 +24,12 @@ #include <test/fuzz/fuzz.h> +void initialize() +{ + // Fuzzers using pubkey must hold an ECCVerifyHandle. + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + void test_one_input(const std::vector<uint8_t>& buffer) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index 9444cd489e..7acdd76857 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -2,12 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <pubkey.h> #include <script/interpreter.h> -#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <util/memory.h> #include <limits> +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index cfa160dde2..da4e623e98 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -4,11 +4,9 @@ #include <test/fuzz/fuzz.h> +#include <cstdint> #include <unistd.h> - -#include <pubkey.h> -#include <util/memory.h> - +#include <vector> static bool read_stdin(std::vector<uint8_t>& data) { @@ -22,9 +20,9 @@ static bool read_stdin(std::vector<uint8_t>& data) return length == 0; } -static void initialize() +// Default initialization: Override using a non-weak initialize(). +__attribute__((weak)) void initialize() { - const static auto verify_handle = MakeUnique<ECCVerifyHandle>(); } // This function is used by libFuzzer @@ -42,13 +40,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -// Disabled under WIN32 due to clash with Cygwin's WinMain. -#ifndef WIN32 // Declare main(...) "weak" to allow for libFuzzer linking. libFuzzer provides // the main(...) function. -__attribute__((weak)) -#endif -int main(int argc, char **argv) +__attribute__((weak)) int main(int argc, char** argv) { initialize(); #ifdef __AFL_INIT diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 573bd572db..3be202b16e 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -8,7 +8,7 @@ #include <stdint.h> #include <vector> - +void initialize(); void test_one_input(const std::vector<uint8_t>& buffer); #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/fuzz/parse_iso8601.cpp b/src/test/fuzz/parse_iso8601.cpp new file mode 100644 index 0000000000..c86f8a853e --- /dev/null +++ b/src/test/fuzz/parse_iso8601.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <util/time.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const int64_t random_time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const std::string random_string = fuzzed_data_provider.ConsumeRemainingBytesAsString(); + + const std::string iso8601_datetime = FormatISO8601DateTime(random_time); + const int64_t parsed_time_1 = ParseISO8601DateTime(iso8601_datetime); + if (random_time >= 0) { + assert(parsed_time_1 >= 0); + if (iso8601_datetime.length() == 20) { + assert(parsed_time_1 == random_time); + } + } + + const int64_t parsed_time_2 = ParseISO8601DateTime(random_string); + assert(parsed_time_2 >= 0); +} diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp new file mode 100644 index 0000000000..0469e87de6 --- /dev/null +++ b/src/test/fuzz/script.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <compressor.h> +#include <core_io.h> +#include <core_memusage.h> +#include <policy/policy.h> +#include <pubkey.h> +#include <script/descriptor.h> +#include <script/script.h> +#include <script/sign.h> +#include <script/signingprovider.h> +#include <script/standard.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <util/memory.h> + +void initialize() +{ + // Fuzzers using pubkey must hold an ECCVerifyHandle. + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const CScript script(buffer.begin(), buffer.end()); + + std::vector<unsigned char> compressed; + (void)CompressScript(script, compressed); + + CTxDestination address; + (void)ExtractDestination(script, address); + + txnouttype type_ret; + std::vector<CTxDestination> addresses; + int required_ret; + (void)ExtractDestinations(script, type_ret, addresses, required_ret); + + (void)GetScriptForWitness(script); + + const FlatSigningProvider signing_provider; + (void)InferDescriptor(script, signing_provider); + + (void)IsSegWitOutput(signing_provider, script); + + (void)IsSolvable(signing_provider, script); + + txnouttype which_type; + (void)IsStandard(script, which_type); + + (void)RecursiveDynamicUsage(script); + + std::vector<std::vector<unsigned char>> solutions; + (void)Solver(script, solutions); + + (void)script.HasValidOps(); + (void)script.IsPayToScriptHash(); + (void)script.IsPayToWitnessScriptHash(); + (void)script.IsPushOnly(); + (void)script.IsUnspendable(); + (void)script.GetSigOpCount(/* fAccurate= */ false); +} diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 0bf5cd5c72..08622d0979 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -2,8 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <pubkey.h> #include <script/interpreter.h> #include <streams.h> +#include <util/memory.h> #include <version.h> #include <test/fuzz/fuzz.h> @@ -11,6 +13,11 @@ /** Flags that are not forbidden by an assert */ static bool IsValidFlagCombination(unsigned flags); +void initialize() +{ + static const auto verify_handle = MakeUnique<ECCVerifyHandle>(); +} + void test_one_input(const std::vector<uint8_t>& buffer) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 96d7947b07..76b230ef3c 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -42,13 +42,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) return; } - CValidationState state_with_dupe_check; - const bool valid_with_dupe_check = CheckTransaction(tx, state_with_dupe_check, /* fCheckDuplicateInputs= */ true); - CValidationState state_without_dupe_check; - const bool valid_without_dupe_check = CheckTransaction(tx, state_without_dupe_check, /* fCheckDuplicateInputs= */ false); - if (valid_with_dupe_check) { - assert(valid_without_dupe_check); - } + TxValidationState state_with_dupe_check; + (void)CheckTransaction(tx, state_with_dupe_check); const CFeeRate dust_relay_fee{DUST_RELAY_TX_FEE}; std::string reason; diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 77304fe918..5abd1087ec 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -4,7 +4,7 @@ #include <util/strencodings.h> #include <util/system.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <utility> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d91fcb0034..b864e6e599 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -6,7 +6,7 @@ #include <crypto/siphash.h> #include <hash.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index e924f27d1b..b52513f4af 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -9,7 +9,7 @@ #include <key_io.h> #include <script/script.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_properties.cpp b/src/test/key_properties.cpp index 95587130fc..0e45a2549d 100644 --- a/src/test/key_properties.cpp +++ b/src/test/key_properties.cpp @@ -4,7 +4,7 @@ #include <key.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 3e99dcaa40..85dc961bea 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -8,7 +8,7 @@ #include <uint256.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <vector> diff --git a/src/test/limitedmap_tests.cpp b/src/test/limitedmap_tests.cpp index 00b36f51fb..ea18debbd3 100644 --- a/src/test/limitedmap_tests.cpp +++ b/src/test/limitedmap_tests.cpp @@ -4,7 +4,7 @@ #include <limitedmap.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp new file mode 100644 index 0000000000..25655b8894 --- /dev/null +++ b/src/test/logging_tests.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <logging/timer.h> +#include <test/util/setup_common.h> + +#include <chrono> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(logging_timer) +{ + + SetMockTime(1); + auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); + + SetMockTime(1); + auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)"); + + SetMockTime(1); + auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); + SetMockTime(2); + BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000.00μs)"); + + SetMockTime(0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index fe5d31b7d3..38fed51af2 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -7,7 +7,7 @@ #include <util/system.h> #include <util/time.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <vector> @@ -749,6 +749,43 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) pool.GetTransactionAncestry(ty6->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, 9ULL); BOOST_CHECK_EQUAL(descendants, 6ULL); + + /* Ancestors represented more than once ("diamond") */ + // + // [ta].0 <- [tb].0 -----<------- [td].0 + // | | + // \---1 <- [tc].0 --<--/ + // + CTransactionRef ta, tb, tc, td; + ta = make_tx(/* output_values */ {10 * COIN}); + tb = make_tx(/* output_values */ {5 * COIN, 3 * COIN}, /* inputs */ {ta}); + tc = make_tx(/* output_values */ {2 * COIN}, /* inputs */ {tb}, /* input_indices */ {1}); + td = make_tx(/* output_values */ {6 * COIN}, /* inputs */ {tb, tc}, /* input_indices */ {0, 0}); + pool.clear(); + pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(tc)); + pool.addUnchecked(entry.Fee(10000LL).FromTx(td)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =================== =========== + // ta 1 (ta 4 (ta,tb,tc,td) + // tb 2 (ta,tb) 4 (ta,tb,tc,td) + // tc 3 (ta,tb,tc) 4 (ta,tb,tc,td) + // td 4 (ta,tb,tc,td) 4 (ta,tb,tc,td) + pool.GetTransactionAncestry(ta->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tb->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tc->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(td->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 4ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index dc38a1a818..03dce552fc 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/merkle.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -345,8 +345,8 @@ BOOST_AUTO_TEST_CASE(merkle_test_BlockWitness) hashes[0].SetNull(); hashes[1] = block.vtx[1]->GetHash(); - uint256 merkelRootofHashes = ComputeMerkleRoot(hashes); + uint256 merkleRootofHashes = ComputeMerkleRoot(hashes); - BOOST_CHECK_EQUAL(merkelRootofHashes, blockWitness); + BOOST_CHECK_EQUAL(merkleRootofHashes, blockWitness); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index eac43471c7..9f8c4ba5c5 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -4,7 +4,7 @@ #include <merkleblock.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index c9661b730d..6ed7350ea2 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -17,7 +17,7 @@ #include <util/time.h> #include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <memory> diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 7c60abb93f..97a918da45 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -11,7 +11,7 @@ #include <script/signingprovider.h> #include <tinyformat.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index fed65afdbf..daf7fea6ad 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -5,7 +5,7 @@ #include <addrdb.h> #include <addrman.h> #include <clientversion.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> #include <serialize.h> @@ -128,9 +128,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); CAddrMan addrman2; - CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); - BOOST_CHECK(adb.Read(addrman2, ssPeers2)); + BOOST_CHECK(CAddrDB::Read(addrman2, ssPeers2)); BOOST_CHECK(addrman2.size() == 3); } @@ -160,9 +159,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); CAddrMan addrman2; - CAddrDB adb; BOOST_CHECK(addrman2.size() == 0); - BOOST_CHECK(!adb.Read(addrman2, ssPeers2)); + BOOST_CHECK(!CAddrDB::Read(addrman2, ssPeers2)); BOOST_CHECK(addrman2.size() == 0); } @@ -303,5 +301,19 @@ BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) BOOST_CHECK_EQUAL(IsLocal(addr), false); } +BOOST_AUTO_TEST_CASE(PoissonNextSend) +{ + g_mock_deterministic_tests = true; + + int64_t now = 5000; + int average_interval_seconds = 600; + + auto poisson = ::PoissonNextSend(now, average_interval_seconds); + std::chrono::microseconds poisson_chrono = ::PoissonNextSend(std::chrono::microseconds{now}, std::chrono::seconds{average_interval_seconds}); + + BOOST_CHECK_EQUAL(poisson, poisson_chrono.count()); + + g_mock_deterministic_tests = false; +} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index a3d0831624..78c11ff202 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -4,7 +4,7 @@ #include <netbase.h> #include <net_permissions.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <string> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index c5513ae9fa..bf58bd63b9 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -9,7 +9,7 @@ #include <uint256.h> #include <arith_uint256.h> #include <version.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -30,7 +30,6 @@ BOOST_FIXTURE_TEST_SUITE(pmt_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(pmt_test1) { - SeedInsecureRand(false); static const unsigned int nTxCounts[] = {1, 4, 7, 17, 56, 100, 127, 256, 312, 513, 1000, 4095}; for (int i = 0; i < 12; i++) { diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 5368f82ffe..025e2b78ca 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -8,7 +8,7 @@ #include <uint256.h> #include <util/time.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 6c99021d97..0f9872f434 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -5,7 +5,7 @@ #include <chain.h> #include <chainparams.h> #include <pow.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index fc1f946bba..9782b78f2c 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -9,7 +9,7 @@ #include <serialize.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 41ca8029e5..04bf7c20c1 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -12,7 +12,7 @@ #include <support/events.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index e6fbe2355d..e0df41a971 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -4,7 +4,7 @@ #include <random.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 69db9dcf4e..532fe143ae 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <reverselock.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 5ae0812243..52dd22de7e 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -7,9 +7,9 @@ #include <rpc/util.h> #include <core_io.h> -#include <init.h> #include <interfaces/chain.h> -#include <test/setup_common.h> +#include <node/context.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <boost/algorithm/string.hpp> @@ -112,14 +112,14 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign) std::string notsigned = r.get_str(); std::string privkey1 = "\"KzsXybp9jX64P5ekX1KUxRQ79Jht9uzW7LorgwE65i5rWACL6LQe\""; std::string privkey2 = "\"Kyhdf5LuKTRx4ge69ybABsiUAWjVRK4XGxAKk2FQLp2HjGMy87Z4\""; - InitInterfaces interfaces; - interfaces.chain = interfaces::MakeChain(); - g_rpc_interfaces = &interfaces; + NodeContext node; + node.chain = interfaces::MakeChain(node); + g_rpc_node = &node; r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" [] "+prevout); BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == false); r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" ["+privkey1+","+privkey2+"] "+prevout); BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == true); - g_rpc_interfaces = nullptr; + g_rpc_node = nullptr; } BOOST_AUTO_TEST_CASE(rpc_createraw_op_return) diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 891aa8e5c3..4d50845256 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -4,7 +4,7 @@ #include <compat/sanity.h> #include <key.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 42242b962b..b292d5b0d0 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -5,7 +5,7 @@ #include <random.h> #include <scheduler.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/thread.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index f451d80984..8c1e843b0b 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -11,7 +11,7 @@ #include <policy/settings.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> @@ -209,20 +209,21 @@ BOOST_AUTO_TEST_CASE(is) p2sh << OP_HASH160 << ToByteVector(dummy) << OP_EQUAL; BOOST_CHECK(p2sh.IsPayToScriptHash()); - // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes: std::vector<unsigned char> direct = {OP_HASH160, 20}; direct.insert(direct.end(), 20, 0); direct.push_back(OP_EQUAL); BOOST_CHECK(CScript(direct.begin(), direct.end()).IsPayToScriptHash()); + + // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes: std::vector<unsigned char> pushdata1 = {OP_HASH160, OP_PUSHDATA1, 20}; pushdata1.insert(pushdata1.end(), 20, 0); pushdata1.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata1.begin(), pushdata1.end()).IsPayToScriptHash()); - std::vector<unsigned char> pushdata2 = {OP_HASH160, 20, 0}; + std::vector<unsigned char> pushdata2 = {OP_HASH160, OP_PUSHDATA2, 20, 0}; pushdata2.insert(pushdata2.end(), 20, 0); pushdata2.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata2.begin(), pushdata2.end()).IsPayToScriptHash()); - std::vector<unsigned char> pushdata4 = {OP_HASH160, 20, 0, 0, 0}; + std::vector<unsigned char> pushdata4 = {OP_HASH160, OP_PUSHDATA4, 20, 0, 0, 0}; pushdata4.insert(pushdata4.end(), 20, 0); pushdata4.push_back(OP_EQUAL); BOOST_CHECK(!CScript(pushdata4.begin(), pushdata4.end()).IsPayToScriptHash()); diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 412a57dd9d..de990d9254 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -6,7 +6,7 @@ #include <script/script.h> #include <script/signingprovider.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 84a70fe78b..26015ca4c2 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -12,7 +12,8 @@ #include <script/signingprovider.h> #include <util/system.h> #include <util/strencodings.h> -#include <test/setup_common.h> +#include <test/util/transaction_utils.h> +#include <test/util/setup_common.h> #include <rpc/util.h> #include <streams.h> @@ -121,40 +122,6 @@ static ScriptError_t ParseScriptError(const std::string &name) BOOST_FIXTURE_TEST_SUITE(script_tests, BasicTestingSetup) -CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue = 0) -{ - CMutableTransaction txCredit; - txCredit.nVersion = 1; - txCredit.nLockTime = 0; - txCredit.vin.resize(1); - txCredit.vout.resize(1); - txCredit.vin[0].prevout.SetNull(); - txCredit.vin[0].scriptSig = CScript() << CScriptNum(0) << CScriptNum(0); - txCredit.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txCredit.vout[0].scriptPubKey = scriptPubKey; - txCredit.vout[0].nValue = nValue; - - return txCredit; -} - -CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit) -{ - CMutableTransaction txSpend; - txSpend.nVersion = 1; - txSpend.nLockTime = 0; - txSpend.vin.resize(1); - txSpend.vout.resize(1); - txSpend.vin[0].scriptWitness = scriptWitness; - txSpend.vin[0].prevout.hash = txCredit.GetHash(); - txSpend.vin[0].prevout.n = 0; - txSpend.vin[0].scriptSig = scriptSig; - txSpend.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txSpend.vout[0].scriptPubKey = CScript(); - txSpend.vout[0].nValue = txCredit.vout[0].nValue; - - return txSpend; -} - void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScriptWitness& scriptWitness, int flags, const std::string& message, int scriptError, CAmount nValue = 0) { bool expect = (scriptError == SCRIPT_ERR_OK); diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index e7916f5000..40a6f69668 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -4,7 +4,7 @@ #include <test/scriptnum10.h> #include <script/script.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <limits.h> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index b90be15fba..303bb9b88c 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -5,7 +5,7 @@ #include <serialize.h> #include <streams.h> #include <hash.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <stdint.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp new file mode 100644 index 0000000000..b0ee76ea6b --- /dev/null +++ b/src/test/settings_tests.cpp @@ -0,0 +1,163 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/settings.h> + +#include <test/util.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> +#include <univalue.h> +#include <util/strencodings.h> +#include <vector> + +BOOST_FIXTURE_TEST_SUITE(settings_tests, BasicTestingSetup) + +//! Check settings struct contents against expected json strings. +static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) +{ + util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false); + util::SettingsValue list_value(util::SettingsValue::VARR); + for (const auto& item : GetSettingsList(settings, "section", "name", false)) { + list_value.push_back(item); + } + BOOST_CHECK_EQUAL(single_value.write().c_str(), single_val); + BOOST_CHECK_EQUAL(list_value.write().c_str(), list_val); +}; + +// Simple settings merge test case. +BOOST_AUTO_TEST_CASE(Simple) +{ + util::Settings settings; + settings.command_line_options["name"].push_back("val1"); + settings.command_line_options["name"].push_back("val2"); + settings.ro_config["section"]["name"].push_back(2); + + // The last given arg takes precedence when specified via commandline. + CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); + + util::Settings settings2; + settings2.ro_config["section"]["name"].push_back("val2"); + settings2.ro_config["section"]["name"].push_back("val3"); + + // The first given arg takes precedence when specified via config file. + CheckValues(settings2, R"("val2")", R"(["val2","val3"])"); +} + +// Test different ways settings can be merged, and verify results. This test can +// be used to confirm that updates to settings code don't change behavior +// unintentionally. +struct MergeTestingSetup : public BasicTestingSetup { + //! Max number of actions to sequence together. Can decrease this when + //! debugging to make test results easier to understand. + static constexpr int MAX_ACTIONS = 3; + + enum Action { END, SET, NEGATE, SECTION_SET, SECTION_NEGATE }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ActionList arg_actions = {}; + // command_line_options do not have sections. Only iterate over SET and NEGATE + ForEachNoDup(arg_actions, SET, NEGATE, [&]{ + ActionList conf_actions = {}; + ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&]{ + for (bool force_set : {false, true}) { + for (bool ignore_default_section_config : {false, true}) { + fn(arg_actions, conf_actions, force_set, ignore_default_section_config); + } + } + }); + }); + } +}; + +// Regression test covering different ways config settings can be merged. The +// test parses and merges settings, representing the results as strings that get +// compared against an expected hash. To debug, the result strings can be dumped +// to a file (see comments below). +BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("SETTINGS_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + const std::string& network = CBaseChainParams::MAIN; + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool force_set, + bool ignore_default_section_config) { + std::string desc; + int value_suffix = 0; + util::Settings settings; + + const std::string& name = ignore_default_section_config ? "wallet" : "server"; + auto push_values = [&](Action action, const char* value_prefix, const std::string& name_prefix, + std::vector<util::SettingsValue>& dest) { + if (action == SET || action == SECTION_SET) { + for (int i = 0; i < 2; ++i) { + dest.push_back(value_prefix + std::to_string(++value_suffix)); + desc += " " + name_prefix + name + "=" + dest.back().get_str(); + } + } else if (action == NEGATE || action == SECTION_NEGATE) { + dest.push_back(false); + desc += " " + name_prefix + "no" + name; + } + }; + + if (force_set) { + settings.forced_settings[name] = "forced"; + desc += " " + name + "=forced"; + } + for (Action arg_action : arg_actions) { + push_values(arg_action, "a", "-", settings.command_line_options[name]); + } + for (Action conf_action : conf_actions) { + bool use_section = conf_action == SECTION_SET || conf_action == SECTION_NEGATE; + push_values(conf_action, "c", use_section ? network + "." : "", + settings.ro_config[use_section ? network : ""][name]); + } + + desc += " || "; + desc += GetSetting(settings, network, name, ignore_default_section_config, /* get_chain_name= */ false).write(); + desc += " |"; + for (const auto& s : GetSettingsList(settings, network, name, ignore_default_section_config)) { + desc += " "; + desc += s.write(); + } + desc += " |"; + if (OnlyHasDefaultSectionSetting(settings, network, name)) desc += " ignored"; + desc += "\n"; + + out_sha.Write((const unsigned char*)desc.data(), desc.size()); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + + // If check below fails, should manually dump the results with: + // + // SETTINGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=settings_tests/Merge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || GetSetting() | GetSettingsList() | OnlyHasDefaultSectionSetting() + BOOST_CHECK_EQUAL(out_sha_hex, "79db02d74e3e193196541b67c068b40ebd0c124a24b3ecbe9cbf7e85b1c4ba7a"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 15f8db899b..2c56bbdbb0 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -10,7 +10,7 @@ #include <script/script.h> #include <serialize.h> #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <util/strencodings.h> #include <version.h> @@ -119,8 +119,6 @@ BOOST_FIXTURE_TEST_SUITE(sighash_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(sighash_test) { - SeedInsecureRand(false); - #if defined(PRINT_SIGHASH_JSON) std::cout << "[\n"; std::cout << "\t[\"raw_transaction, script, input_index, hashType, signature_hash (result)\"],\n"; @@ -193,7 +191,7 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) CDataStream stream(ParseHex(raw_tx), SER_NETWORK, PROTOCOL_VERSION); stream >> tx; - CValidationState state; + TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); BOOST_CHECK(state.IsValid()); diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index a32f2cda92..6462fcefe3 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -9,7 +9,7 @@ #include <script/script.h> #include <script/standard.h> #include <uint256.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 1cba3a1297..7ede79279f 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 6075fbfeca..177d8fda73 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <streams.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -338,7 +338,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) { // Make this test deterministic. - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); for (int rep = 0; rep < 50; ++rep) { FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index c1399d2dbe..188e9986ff 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <sync.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 7b00222ab7..4721151197 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -5,7 +5,8 @@ #include <netaddress.h> #include <noui.h> -#include <test/setup_common.h> +#include <test/util/logging.h> +#include <test/util/setup_common.h> #include <timedata.h> #include <warnings.h> @@ -59,9 +60,10 @@ BOOST_AUTO_TEST_CASE(addtimedata) MultiAddTimeData(3, DEFAULT_MAX_TIME_ADJUSTMENT + 1); // Filter size is 1 + 3 = 4: It is always initialized with a single element (offset 0) - noui_suppress(); - MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 - noui_reconnect(); + { + ASSERT_DEBUG_LOG("Please check that your computer's date and time are correct!"); + MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 + } BOOST_CHECK(GetWarnings("gui").find("clock is wrong") != std::string::npos); diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp index d846062d9b..41aa17988c 100644 --- a/src/test/torcontrol_tests.cpp +++ b/src/test/torcontrol_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 34192c6b6a..eb0050a4a3 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -4,7 +4,7 @@ #include <test/data/tx_invalid.json.h> #include <test/data/tx_valid.json.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <clientversion.h> #include <checkqueue.h> @@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); CTransaction tx(deserialize, stream); - CValidationState state; + TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); BOOST_CHECK(state.IsValid()); @@ -239,7 +239,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); CTransaction tx(deserialize, stream); - CValidationState state; + TxValidationState state; fValid = CheckTransaction(tx, state) && state.IsValid(); PrecomputedTransactionData txdata(tx); @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_CASE(basic_transaction_tests) CDataStream stream(vch, SER_DISK, CLIENT_VERSION); CMutableTransaction tx; stream >> tx; - CValidationState state; + TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(CTransaction(tx), state) && state.IsValid(), "Simple deserialized transaction should be valid."); // Check that duplicate txins fail @@ -706,7 +706,9 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) BOOST_CHECK_EQUAL(nDustThreshold, 546); // dust: t.vout[0].nValue = nDustThreshold - 1; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "dust"); // not dust: t.vout[0].nValue = nDustThreshold; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); @@ -716,14 +718,18 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) dustRelayFee = CFeeRate(3702); // dust: t.vout[0].nValue = 673 - 1; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "dust"); // not dust: t.vout[0].nValue = 673; BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); t.vout[0].scriptPubKey = CScript() << OP_1; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptpubkey"); // MAX_OP_RETURN_RELAY-byte TX_NULL_DATA (standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); @@ -733,7 +739,9 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // MAX_OP_RETURN_RELAY+1-byte TX_NULL_DATA (non-standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY + 1, t.vout[0].scriptPubKey.size()); + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptpubkey"); // Data payload can be encoded in any way... t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex(""); @@ -748,7 +756,9 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // ...so long as it only contains PUSHDATA's t.vout[0].scriptPubKey = CScript() << OP_RETURN << OP_RETURN; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptpubkey"); // TX_NULL_DATA w/o PUSHDATA t.vout.resize(1); @@ -759,15 +769,34 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout.resize(2); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "multi-op-return"); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "multi-op-return"); t.vout[0].scriptPubKey = CScript() << OP_RETURN; t.vout[1].scriptPubKey = CScript() << OP_RETURN; + reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "multi-op-return"); + + // Check large scriptSig (non-standard if size is >1650 bytes) + t.vout.resize(1); + t.vout[0].nValue = MAX_MONEY; + t.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + // OP_PUSHDATA2 with len (3 bytes) + data (1647 bytes) = 1650 bytes + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1647, 0); // 1650 + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + + t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1648, 0); // 1651 + reason.clear(); + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason, "scriptsig-size"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 0ac4b7ebc9..4b0214a15a 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -5,7 +5,7 @@ #include <chainparams.h> #include <index/txindex.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index 2356e0ccdc..245c03d774 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -6,7 +6,7 @@ #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/script.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> @@ -30,7 +30,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK(CTransaction(coinbaseTx).IsCoinBase()); - CValidationState state; + TxValidationState state; LOCK(cs_main); @@ -39,7 +39,6 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK_EQUAL( false, AcceptToMemoryPool(mempool, state, MakeTransactionRef(coinbaseTx), - nullptr /* pfMissingInputs */, nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */)); @@ -50,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) // Check that the validation state reflects the unsuccessful attempt. BOOST_CHECK(state.IsInvalid()); BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); - BOOST_CHECK(state.GetReason() == ValidationInvalidReason::CONSENSUS); + BOOST_CHECK(state.GetResult() == TxValidationResult::TX_CONSENSUS); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 193858cca9..a5bc15bb9f 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -9,11 +9,11 @@ #include <script/standard.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); +bool CheckInputs(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); BOOST_AUTO_TEST_SUITE(tx_validationcache_tests) @@ -22,8 +22,8 @@ ToMemPool(const CMutableTransaction& tx) { LOCK(cs_main); - CValidationState state; - return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), nullptr /* pfMissingInputs */, + TxValidationState state; + return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */); } @@ -114,7 +114,7 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail // If we add many more flags, this loop can get too expensive, but we can // rewrite in the future to randomly pick a set of flags to evaluate. for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { - CValidationState state; + TxValidationState state; // Filter out incompatible flag choices if ((test_flags & SCRIPT_VERIFY_CLEANSTACK)) { // CLEANSTACK requires P2SH and WITNESS, see VerifyScript() in @@ -201,7 +201,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) { LOCK(cs_main); - CValidationState state; + TxValidationState state; PrecomputedTransactionData ptd_spend_tx(spend_tx); BOOST_CHECK(!CheckInputs(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); @@ -270,7 +270,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; - CValidationState state; + TxValidationState state; PrecomputedTransactionData txdata(invalid_with_cltv_tx); BOOST_CHECK(CheckInputs(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } @@ -298,7 +298,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; - CValidationState state; + TxValidationState state; PrecomputedTransactionData txdata(invalid_with_csv_tx); BOOST_CHECK(CheckInputs(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } @@ -359,7 +359,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Invalidate vin[1] tx.vin[1].scriptWitness.SetNull(); - CValidationState state; + TxValidationState state; PrecomputedTransactionData txdata(tx); // This transaction is now invalid under segwit, because of the second input. BOOST_CHECK(!CheckInputs(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 33a118c2bb..7293ecd325 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -6,7 +6,7 @@ #include <streams.h> #include <uint256.h> #include <version.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> #include <sstream> diff --git a/src/test/util.cpp b/src/test/util.cpp index b7bb6deeaa..ed031270f2 100644 --- a/src/test/util.cpp +++ b/src/test/util.cpp @@ -32,13 +32,15 @@ std::string getnewaddress(CWallet& w) void importaddress(CWallet& wallet, const std::string& address) { + auto spk_man = wallet.GetLegacyScriptPubKeyMan(); LOCK(wallet.cs_wallet); + AssertLockHeld(spk_man->cs_wallet); const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); const auto script = GetScriptForDestination(dest); wallet.MarkDirty(); - assert(!wallet.HaveWatchOnly(script)); - if (!wallet.AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); + assert(!spk_man->HaveWatchOnly(script)); + if (!spk_man->AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); wallet.SetAddressBook(dest, /* label */ "", "receive"); } #endif // ENABLE_WALLET diff --git a/src/test/util/README.md b/src/test/util/README.md new file mode 100644 index 0000000000..36ad645201 --- /dev/null +++ b/src/test/util/README.md @@ -0,0 +1,11 @@ +# Test library + +This contains files for the test library, which is used by the test binaries (unit tests, benchmarks, fuzzers, gui +tests). + +Generally, the files in this folder should be well-separated modules. New code should be added to existing modules or +(when in doubt) a new module should be created. + +The utilities in here are compiled into a library, which does not hold any state. However, the main file `setup_common` +defines the common test setup for all test binaries. The test binaries will handle the global state when they +instantiate the `BasicTestingSetup` (or one of its derived classes). diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp new file mode 100644 index 0000000000..bccff5e5a6 --- /dev/null +++ b/src/test/util/blockfilter.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/blockfilter.h> + +#include <chainparams.h> +#include <validation.h> + + +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) +{ + CBlock block; + if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + return false; + } + + CBlockUndo block_undo; + if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + return false; + } + + filter = BlockFilter(filter_type, block, block_undo); + return true; +} + diff --git a/src/test/util/blockfilter.h b/src/test/util/blockfilter.h new file mode 100644 index 0000000000..79d11dcad8 --- /dev/null +++ b/src/test/util/blockfilter.h @@ -0,0 +1,13 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_BLOCKFILTER_H +#define BITCOIN_TEST_UTIL_BLOCKFILTER_H + +#include <blockfilter.h> +class CBlockIndex; + +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter); + +#endif // BITCOIN_TEST_UTIL_BLOCKFILTER_H diff --git a/src/test/util/logging.cpp b/src/test/util/logging.cpp new file mode 100644 index 0000000000..fe2e69104b --- /dev/null +++ b/src/test/util/logging.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/logging.h> + +#include <logging.h> +#include <noui.h> +#include <tinyformat.h> +#include <util/memory.h> + +#include <stdexcept> + +DebugLogHelper::DebugLogHelper(std::string message) + : m_message{std::move(message)} +{ + m_print_connection = LogInstance().PushBackCallback( + [this](const std::string& s) { + if (m_found) return; + m_found = s.find(m_message) != std::string::npos; + }); + noui_test_redirect(); +} + +void DebugLogHelper::check_found() +{ + noui_reconnect(); + LogInstance().DeleteCallback(m_print_connection); + if (!m_found) { + throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); + } +} diff --git a/src/test/util/logging.h b/src/test/util/logging.h new file mode 100644 index 0000000000..45ec44173c --- /dev/null +++ b/src/test/util/logging.h @@ -0,0 +1,29 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_LOGGING_H +#define BITCOIN_TEST_UTIL_LOGGING_H + +#include <util/macros.h> + +#include <functional> +#include <list> +#include <string> + +class DebugLogHelper +{ + const std::string m_message; + bool m_found{false}; + std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + + void check_found(); + +public: + DebugLogHelper(std::string message); + ~DebugLogHelper() { check_found(); } +}; + +#define ASSERT_DEBUG_LOG(message) DebugLogHelper PASTE2(debugloghelper, __COUNTER__)(message) + +#endif // BITCOIN_TEST_UTIL_LOGGING_H diff --git a/src/test/setup_common.cpp b/src/test/util/setup_common.cpp index bbdf1ef830..0c6ecdf69d 100644 --- a/src/test/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <banman.h> #include <chainparams.h> @@ -15,6 +15,7 @@ #include <net.h> #include <noui.h> #include <pow.h> +#include <rpc/blockchain.h> #include <rpc/register.h> #include <rpc/server.h> #include <script/sigcache.h> @@ -33,6 +34,27 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; FastRandomContext g_insecure_rand_ctx; +/** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ +static FastRandomContext g_insecure_rand_ctx_temp_path; + +/** Return the unsigned from the environment var if available, otherwise 0 */ +static uint256 GetUintFromEnv(const std::string& env_name) +{ + const char* num = std::getenv(env_name.c_str()); + if (!num) return {}; + return uint256S(num); +} + +void Seed(FastRandomContext& ctx) +{ + // Should be enough to get the seed once for the process + static uint256 seed{}; + static const std::string RANDOM_CTX_SEED{"RANDOM_CTX_SEED"}; + if (seed.IsNull()) seed = GetUintFromEnv(RANDOM_CTX_SEED); + if (seed.IsNull()) seed = GetRandHash(); + LogPrintf("%s: Setting random seed for current tests to %s=%s\n", __func__, RANDOM_CTX_SEED, seed.GetHex()); + ctx = FastRandomContext(seed); +} std::ostream& operator<<(std::ostream& os, const uint256& num) { @@ -41,12 +63,13 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) } BasicTestingSetup::BasicTestingSetup(const std::string& chainName) - : m_path_root(fs::temp_directory_path() / "test_common_" PACKAGE_NAME / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30)))) + : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / std::to_string(g_insecure_rand_ctx_temp_path.rand32())} { fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); SelectParams(chainName); + SeedInsecureRand(); gArgs.ForceSetArg("-printtoconsole", "0"); InitLogging(); LogInstance().StartLogging(); @@ -76,6 +99,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework // instead of unit tests, but for now we need these here. + g_rpc_node = &m_node; RegisterAllCoreRPCCommands(tableRPC); // We have to run a scheduler thread to prevent ActivateBestChain @@ -95,17 +119,20 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha throw std::runtime_error("LoadGenesisBlock failed."); } - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, chainparams)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); } - nScriptCheckThreads = 3; - for (int i = 0; i < nScriptCheckThreads - 1; i++) + // Start script-checking threads. Set g_parallel_script_checks to true so they are used. + constexpr int script_check_threads = 2; + for (int i = 0; i < script_check_threads; ++i) { threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); + } + g_parallel_script_checks = true; - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. + m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); + m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. } TestingSetup::~TestingSetup() @@ -114,18 +141,20 @@ TestingSetup::~TestingSetup() threadGroup.join_all(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - g_connman.reset(); - g_banman.reset(); + g_rpc_node = nullptr; + m_node.connman.reset(); + m_node.banman.reset(); UnloadBlockIndex(); g_chainstate.reset(); pblocktree.reset(); } -TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) +TestChain100Setup::TestChain100Setup() { // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. // TODO: fix the code to support SegWit blocks. gArgs.ForceSetArg("-segwitheight", "432"); + // Need to recreate chainparams SelectParams(CBaseChainParams::REGTEST); // Generate a 100-block chain: @@ -139,12 +168,9 @@ TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) } } -// // Create a new block with just given transactions, coinbase paying to // scriptPubKey, and try to add it to the current chain. -// -CBlock -TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) +CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); @@ -172,6 +198,7 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& TestChain100Setup::~TestChain100Setup() { + gArgs.ForceSetArg("-segwitheight", "0"); } diff --git a/src/test/setup_common.h b/src/test/util/setup_common.h index 6c9494898c..1e2e059a56 100644 --- a/src/test/setup_common.h +++ b/src/test/util/setup_common.h @@ -2,12 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_TEST_SETUP_COMMON_H -#define BITCOIN_TEST_SETUP_COMMON_H +#ifndef BITCOIN_TEST_UTIL_SETUP_COMMON_H +#define BITCOIN_TEST_UTIL_SETUP_COMMON_H #include <chainparamsbase.h> #include <fs.h> #include <key.h> +#include <node/context.h> #include <pubkey.h> #include <random.h> #include <scheduler.h> @@ -38,9 +39,21 @@ extern FastRandomContext g_insecure_rand_ctx; */ extern bool g_mock_deterministic_tests; -static inline void SeedInsecureRand(bool deterministic = false) +enum class SeedRand { + ZEROS, //!< Seed with a compile time constant of zeros + SEED, //!< Call the Seed() helper +}; + +/** Seed the given random ctx or use the seed passed in via an environment var */ +void Seed(FastRandomContext& ctx); + +static inline void SeedInsecureRand(SeedRand seed = SeedRand::SEED) { - g_insecure_rand_ctx = FastRandomContext(deterministic); + if (seed == SeedRand::ZEROS) { + g_insecure_rand_ctx = FastRandomContext(/* deterministic */ true); + } else { + Seed(g_insecure_rand_ctx); + } } static inline uint32_t InsecureRand32() { return g_insecure_rand_ctx.rand32(); } @@ -67,6 +80,7 @@ private: * Included are coins database, script check threads setup. */ struct TestingSetup : public BasicTestingSetup { + NodeContext m_node; boost::thread_group threadGroup; CScheduler scheduler; @@ -74,6 +88,12 @@ struct TestingSetup : public BasicTestingSetup { ~TestingSetup(); }; +/** Identical to TestingSetup, but chain set to regtest */ +struct RegTestingSetup : public TestingSetup { + RegTestingSetup() + : TestingSetup{CBaseChainParams::REGTEST} {} +}; + class CBlock; struct CMutableTransaction; class CScript; @@ -82,7 +102,7 @@ class CScript; // Testing fixture that pre-creates a // 100-block REGTEST-mode block chain // -struct TestChain100Setup : public TestingSetup { +struct TestChain100Setup : public RegTestingSetup { TestChain100Setup(); // Create a new block with just given transactions, coinbase paying to diff --git a/src/test/util/str.cpp b/src/test/util/str.cpp new file mode 100644 index 0000000000..c517fe44d9 --- /dev/null +++ b/src/test/util/str.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/str.h> + +#include <cstdint> +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2) +{ + if (s1.size() != s2.size()) return false; + for (size_t i = 0; i < s1.size(); ++i) { + char c1 = s1[i]; + if (c1 >= 'A' && c1 <= 'Z') c1 -= ('A' - 'a'); + char c2 = s2[i]; + if (c2 >= 'A' && c2 <= 'Z') c2 -= ('A' - 'a'); + if (c1 != c2) return false; + } + return true; +} diff --git a/src/test/util/str.h b/src/test/util/str.h new file mode 100644 index 0000000000..63629501e8 --- /dev/null +++ b/src/test/util/str.h @@ -0,0 +1,12 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_STR_H +#define BITCOIN_TEST_UTIL_STR_H + +#include <string> + +bool CaseInsensitiveEqual(const std::string& s1, const std::string& s2); + +#endif // BITCOIN_TEST_UTIL_STR_H diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp new file mode 100644 index 0000000000..90b78effb0 --- /dev/null +++ b/src/test/util/transaction_utils.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/transaction_utils.h> + +CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) +{ + CMutableTransaction txCredit; + txCredit.nVersion = 1; + txCredit.nLockTime = 0; + txCredit.vin.resize(1); + txCredit.vout.resize(1); + txCredit.vin[0].prevout.SetNull(); + txCredit.vin[0].scriptSig = CScript() << CScriptNum(0) << CScriptNum(0); + txCredit.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; + txCredit.vout[0].scriptPubKey = scriptPubKey; + txCredit.vout[0].nValue = nValue; + + return txCredit; +} + +CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit) +{ + CMutableTransaction txSpend; + txSpend.nVersion = 1; + txSpend.nLockTime = 0; + txSpend.vin.resize(1); + txSpend.vout.resize(1); + txSpend.vin[0].scriptWitness = scriptWitness; + txSpend.vin[0].prevout.hash = txCredit.GetHash(); + txSpend.vin[0].prevout.n = 0; + txSpend.vin[0].scriptSig = scriptSig; + txSpend.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; + txSpend.vout[0].scriptPubKey = CScript(); + txSpend.vout[0].nValue = txCredit.vout[0].nValue; + + return txSpend; +} diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h new file mode 100644 index 0000000000..57604646e7 --- /dev/null +++ b/src/test/util/transaction_utils.h @@ -0,0 +1,19 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H +#define BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H + +#include <primitives/transaction.h> + +// create crediting transaction +// [1 coinbase input => 1 output with given scriptPubkey and value] +CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue = 0); + +// create spending transaction +// [1 input with referenced transaction outpoint, scriptSig, scriptWitness => +// 1 output with empty scriptPubKey, full value of referenced transaction] +CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, const CTransaction& txCredit); + +#endif // BITCOIN_TEST_UTIL_TRANSACTION_UTILS_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 02303d0f65..b9fcd97a8f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -6,7 +6,7 @@ #include <clientversion.h> #include <sync.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <test/util.h> #include <util/moneystr.h> #include <util/strencodings.h> @@ -17,6 +17,7 @@ #include <stdint.h> #include <thread> +#include <univalue.h> #include <utility> #include <vector> #ifndef WIN32 @@ -145,9 +146,17 @@ BOOST_AUTO_TEST_CASE(util_Join) BOOST_CHECK_EQUAL(Join<std::string>({"foo", "bar"}, ", ", op_upper), "FOO, BAR"); } -BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) +BOOST_AUTO_TEST_CASE(util_FormatParseISO8601DateTime) { BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); + BOOST_CHECK_EQUAL(FormatISO8601DateTime(0), "1970-01-01T00:00:00Z"); + + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1970-01-01T00:00:00Z"), 0); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("1960-01-01T00:00:00Z"), 0); + BOOST_CHECK_EQUAL(ParseISO8601DateTime("2011-09-30T23:36:17Z"), 1317425777); + + auto time = GetSystemTimeInSeconds(); + BOOST_CHECK_EQUAL(ParseISO8601DateTime(FormatISO8601DateTime(time)), time); } BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) @@ -158,14 +167,12 @@ BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) struct TestArgsManager : public ArgsManager { TestArgsManager() { m_network_only_args.clear(); } - std::map<std::string, std::vector<std::string> >& GetOverrideArgs() { return m_override_args; } - std::map<std::string, std::vector<std::string> >& GetConfigArgs() { return m_config_args; } void ReadConfigString(const std::string str_config) { std::istringstream streamConfig(str_config); { LOCK(cs_args); - m_config_args.clear(); + m_settings.ro_config.clear(); m_config_sections.clear(); } std::string error; @@ -185,6 +192,7 @@ struct TestArgsManager : public ArgsManager using ArgsManager::ReadConfigStream; using ArgsManager::cs_args; using ArgsManager::m_network; + using ArgsManager::m_settings; }; BOOST_AUTO_TEST_CASE(util_ParseParameters) @@ -198,31 +206,86 @@ BOOST_AUTO_TEST_CASE(util_ParseParameters) const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; std::string error; + LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, ccc, d}); BOOST_CHECK(testArgs.ParseParameters(0, (char**)argv_test, error)); - BOOST_CHECK(testArgs.GetOverrideArgs().empty() && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); - BOOST_CHECK(testArgs.GetOverrideArgs().empty() && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(testArgs.GetOverrideArgs().size() == 3 && testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 3 && testArgs.m_settings.ro_config.empty()); BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); - BOOST_CHECK(testArgs.GetOverrideArgs().count("-a") && testArgs.GetOverrideArgs().count("-b") && testArgs.GetOverrideArgs().count("-ccc") - && !testArgs.GetOverrideArgs().count("f") && !testArgs.GetOverrideArgs().count("-d")); - - BOOST_CHECK(testArgs.GetOverrideArgs()["-a"].size() == 1); - BOOST_CHECK(testArgs.GetOverrideArgs()["-a"].front() == ""); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].size() == 2); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].front() == "argument"); - BOOST_CHECK(testArgs.GetOverrideArgs()["-ccc"].back() == "multiple"); + BOOST_CHECK(testArgs.m_settings.command_line_options.count("a") && testArgs.m_settings.command_line_options.count("b") && testArgs.m_settings.command_line_options.count("ccc") + && !testArgs.m_settings.command_line_options.count("f") && !testArgs.m_settings.command_line_options.count("d")); + + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].size() == 1); + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].front().get_str() == ""); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].size() == 2); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].front().get_str() == "argument"); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].back().get_str() == "multiple"); BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } +static void TestParse(const std::string& str, bool expected_bool, int64_t expected_int) +{ + TestArgsManager test; + test.SetupArgs({{"-value", ArgsManager::ALLOW_ANY}}); + std::string arg = "-value=" + str; + const char* argv[] = {"ignored", arg.c_str()}; + std::string error; + BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); + BOOST_CHECK_EQUAL(test.GetArg("-value", 99998), expected_int); + BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), expected_int); +} + +// Test bool and int parsing. +BOOST_AUTO_TEST_CASE(util_ArgParsing) +{ + // Some of these cases could be ambiguous or surprising to users, and might + // be worth triggering errors or warnings in the future. But for now basic + // test coverage is useful to avoid breaking backwards compatibility + // unintentionally. + TestParse("", true, 0); + TestParse(" ", false, 0); + TestParse("0", false, 0); + TestParse("0 ", false, 0); + TestParse(" 0", false, 0); + TestParse("+0", false, 0); + TestParse("-0", false, 0); + TestParse("5", true, 5); + TestParse("5 ", true, 5); + TestParse(" 5", true, 5); + TestParse("+5", true, 5); + TestParse("-5", true, -5); + TestParse("0 5", false, 0); + TestParse("5 0", true, 5); + TestParse("050", true, 50); + TestParse("0.", false, 0); + TestParse("5.", true, 5); + TestParse("0.0", false, 0); + TestParse("0.5", false, 0); + TestParse("5.0", true, 5); + TestParse("5.5", true, 5); + TestParse("x", false, 0); + TestParse("x0", false, 0); + TestParse("x5", false, 0); + TestParse("0x", false, 0); + TestParse("5x", true, 5); + TestParse("0x5", false, 0); + TestParse("false", false, 0); + TestParse("true", false, 0); + TestParse("yes", false, 0); + TestParse("no", false, 0); +} + BOOST_AUTO_TEST_CASE(util_GetBoolArg) { TestArgsManager testArgs; @@ -236,6 +299,7 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArg) const char *argv_test[] = { "ignored", "-a", "-nob", "-c=0", "-d=1", "-e=false", "-f=true"}; std::string error; + LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, c, d, e, f}); BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); @@ -244,8 +308,8 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArg) BOOST_CHECK(testArgs.IsArgSet({'-', opt}) || !opt); // Nothing else should be in the map - BOOST_CHECK(testArgs.GetOverrideArgs().size() == 6 && - testArgs.GetConfigArgs().empty()); + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 6 && + testArgs.m_settings.ro_config.empty()); // The -no prefix should get stripped on the way in. BOOST_CHECK(!testArgs.IsArgSet("-nob")); @@ -341,6 +405,7 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) "iii=2\n"; TestArgsManager test_args; + LOCK(test_args.cs_args); const auto a = std::make_pair("-a", ArgsManager::ALLOW_BOOL); const auto b = std::make_pair("-b", ArgsManager::ALLOW_BOOL); const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_STRING); @@ -357,22 +422,25 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) // expectation: a, b, ccc, d, fff, ggg, h, i end up in map // so do sec1.ccc, sec1.d, sec1.h, sec2.ccc, sec2.iii - BOOST_CHECK(test_args.GetOverrideArgs().empty()); - BOOST_CHECK(test_args.GetConfigArgs().size() == 13); - - BOOST_CHECK(test_args.GetConfigArgs().count("-a") - && test_args.GetConfigArgs().count("-b") - && test_args.GetConfigArgs().count("-ccc") - && test_args.GetConfigArgs().count("-d") - && test_args.GetConfigArgs().count("-fff") - && test_args.GetConfigArgs().count("-ggg") - && test_args.GetConfigArgs().count("-h") - && test_args.GetConfigArgs().count("-i") + BOOST_CHECK(test_args.m_settings.command_line_options.empty()); + BOOST_CHECK(test_args.m_settings.ro_config.size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config[""].size() == 8); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].size() == 2); + + BOOST_CHECK(test_args.m_settings.ro_config[""].count("a") + && test_args.m_settings.ro_config[""].count("b") + && test_args.m_settings.ro_config[""].count("ccc") + && test_args.m_settings.ro_config[""].count("d") + && test_args.m_settings.ro_config[""].count("fff") + && test_args.m_settings.ro_config[""].count("ggg") + && test_args.m_settings.ro_config[""].count("h") + && test_args.m_settings.ro_config[""].count("i") ); - BOOST_CHECK(test_args.GetConfigArgs().count("-sec1.ccc") - && test_args.GetConfigArgs().count("-sec1.h") - && test_args.GetConfigArgs().count("-sec2.ccc") - && test_args.GetConfigArgs().count("-sec2.iii") + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc") + && test_args.m_settings.ro_config["sec1"].count("h") + && test_args.m_settings.ro_config["sec2"].count("ccc") + && test_args.m_settings.ro_config["sec2"].count("iii") ); BOOST_CHECK(test_args.IsArgSet("-a") @@ -511,24 +579,25 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) BOOST_AUTO_TEST_CASE(util_GetArg) { TestArgsManager testArgs; - testArgs.GetOverrideArgs().clear(); - testArgs.GetOverrideArgs()["strtest1"] = {"string..."}; + LOCK(testArgs.cs_args); + testArgs.m_settings.command_line_options.clear(); + testArgs.m_settings.command_line_options["strtest1"] = {"string..."}; // strtest2 undefined on purpose - testArgs.GetOverrideArgs()["inttest1"] = {"12345"}; - testArgs.GetOverrideArgs()["inttest2"] = {"81985529216486895"}; + testArgs.m_settings.command_line_options["inttest1"] = {"12345"}; + testArgs.m_settings.command_line_options["inttest2"] = {"81985529216486895"}; // inttest3 undefined on purpose - testArgs.GetOverrideArgs()["booltest1"] = {""}; + testArgs.m_settings.command_line_options["booltest1"] = {""}; // booltest2 undefined on purpose - testArgs.GetOverrideArgs()["booltest3"] = {"0"}; - testArgs.GetOverrideArgs()["booltest4"] = {"1"}; + testArgs.m_settings.command_line_options["booltest3"] = {"0"}; + testArgs.m_settings.command_line_options["booltest4"] = {"1"}; // priorities - testArgs.GetOverrideArgs()["pritest1"] = {"a", "b"}; - testArgs.GetConfigArgs()["pritest2"] = {"a", "b"}; - testArgs.GetOverrideArgs()["pritest3"] = {"a"}; - testArgs.GetConfigArgs()["pritest3"] = {"b"}; - testArgs.GetOverrideArgs()["pritest4"] = {"a","b"}; - testArgs.GetConfigArgs()["pritest4"] = {"c","d"}; + testArgs.m_settings.command_line_options["pritest1"] = {"a", "b"}; + testArgs.m_settings.ro_config[""]["pritest2"] = {"a", "b"}; + testArgs.m_settings.command_line_options["pritest3"] = {"a"}; + testArgs.m_settings.ro_config[""]["pritest3"] = {"b"}; + testArgs.m_settings.command_line_options["pritest4"] = {"a","b"}; + testArgs.m_settings.ro_config[""]["pritest4"] = {"c","d"}; BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); @@ -882,6 +951,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) desc += " "; desc += argstr + 1; conf += argstr + 1; + conf += "\n"; } std::istringstream conf_stream(conf); BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); @@ -920,7 +990,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) // Results file is formatted like: // // <input> || <output> - BOOST_CHECK_EQUAL(out_sha_hex, "94b4ad55c8ac639a56b93e36f7e32e4c611fd7d7dd7b2be6a71707b1eadcaec7"); + BOOST_CHECK_EQUAL(out_sha_hex, "f0b3a3c29869edc765d579c928f7f1690a71fbb673b49ccf39cbc4de18156a0d"); } BOOST_AUTO_TEST_CASE(util_FormatMoney) @@ -1043,7 +1113,7 @@ BOOST_AUTO_TEST_CASE(util_IsHexNumber) BOOST_AUTO_TEST_CASE(util_seed_insecure_rand) { - SeedInsecureRand(true); + SeedInsecureRand(SeedRand::ZEROS); for (int mod=2;mod<11;mod++) { int mask = 1; diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 71c0168ca3..33f840d9c5 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <util/threadnames.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <thread> #include <vector> diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index b3368d44b6..b7cf82906a 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -11,20 +11,16 @@ #include <pow.h> #include <random.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/time.h> #include <validation.h> #include <validationinterface.h> #include <thread> -struct RegtestingSetup : public TestingSetup { - RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {} -}; - static const std::vector<unsigned char> V_OP_TRUE{OP_TRUE}; -BOOST_FIXTURE_TEST_SUITE(validation_block_tests, RegtestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_block_tests, RegTestingSetup) struct TestSubscriber : public CValidationInterface { uint256 m_expected_tip; @@ -44,9 +40,10 @@ struct TestSubscriber : public CValidationInterface { m_expected_tip = block->GetHash(); } - void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override + void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override { BOOST_CHECK_EQUAL(m_expected_tip, block->GetHash()); + BOOST_CHECK_EQUAL(m_expected_tip, pindex->GetBlockHash()); m_expected_tip = block->hashPrevBlock; } @@ -151,7 +148,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) } bool ignored; - CValidationState state; + BlockValidationState state; std::vector<CBlockHeader> headers; std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); @@ -278,14 +275,13 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Add the txs to the tx pool { LOCK(cs_main); - CValidationState state; + TxValidationState state; std::list<CTransactionRef> plTxnReplaced; for (const auto& tx : txs) { BOOST_REQUIRE(AcceptToMemoryPool( ::mempool, state, tx, - /* pfMissingInputs */ &ignored, &plTxnReplaced, /* bypass_limits */ false, /* nAbsurdFee */ 0)); diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index 101025d31e..3b961db52d 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -6,7 +6,7 @@ #include <net.h> #include <validation.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 0ca3a17974..7b59d539a6 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -4,7 +4,7 @@ #include <chain.h> #include <versionbits.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <chainparams.h> #include <validation.h> #include <consensus/params.h> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index e4c1fd4bc6..08f935c24f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -8,6 +8,7 @@ #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> +#include <optional.h> #include <validation.h> #include <policy/policy.h> #include <policy/fees.h> @@ -155,7 +156,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr // GetMemPoolParents() is only valid for entries in the mempool, so we // iterate mapTx to find parents. for (unsigned int i = 0; i < tx.vin.size(); i++) { - boost::optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); + Optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); if (piter) { parentHashes.insert(*piter); if (parentHashes.size() + 1 > limitAncestorCount) { @@ -591,9 +592,9 @@ void CTxMemPool::clear() static void CheckInputsAndUpdateCoins(const CTransaction& tx, CCoinsViewCache& mempoolDuplicate, const int64_t spendheight) { - CValidationState state; + TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass CAmount txfee = 0; - bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, state, mempoolDuplicate, spendheight, txfee); + bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, std::numeric_limits<int>::max()); } @@ -860,11 +861,11 @@ const CTransaction* CTxMemPool::GetConflictTx(const COutPoint& prevout) const return it == mapNextTx.end() ? nullptr : it->second; } -boost::optional<CTxMemPool::txiter> CTxMemPool::GetIter(const uint256& txid) const +Optional<CTxMemPool::txiter> CTxMemPool::GetIter(const uint256& txid) const { auto it = mapTx.find(txid); if (it != mapTx.end()) return it; - return boost::optional<txiter>{}; + return Optional<txiter>{}; } CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set<uint256>& hashes) const diff --git a/src/txmempool.h b/src/txmempool.h index b51e800001..4a7640b78a 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -17,6 +17,7 @@ #include <coins.h> #include <crypto/siphash.h> #include <indirectmap.h> +#include <optional.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <sync.h> @@ -49,8 +50,6 @@ struct LockPoints LockPoints() : height(0), time(0), maxInputBlock(nullptr) { } }; -class CTxMemPool; - /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well @@ -602,7 +601,7 @@ public: const CTransaction* GetConflictTx(const COutPoint& prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Returns an iterator to the given hash, if found */ - boost::optional<txiter> GetIter(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); + Optional<txiter> GetIter(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Translate a set of hashes into a set of pool iterators to avoid repeated lookups */ setEntries GetIterSet(const std::set<uint256>& hashes) const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/univalue/README.md b/src/univalue/README.md index 36aa786a4c..7c62c33970 100644 --- a/src/univalue/README.md +++ b/src/univalue/README.md @@ -12,21 +12,10 @@ an arbitrary depth. This class is aligned with the JSON standard, [RFC 7159](https://tools.ietf.org/html/rfc7159.html). -## Installation +## Library usage -This project is a standard GNU -[autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html) -project. Build and install instructions are available in the `INSTALL` -file provided with GNU autotools. - -``` -$ ./autogen.sh -$ ./configure -$ make -``` - -## Design - -UniValue provides a single dynamic RAII C++ object class, -and minimizes template use (contra json_spirit). +This is a fork of univalue used by Bitcoin Core. It is not maintained for usage +by other projects. Notably, the API may break in non-backward-compatible ways. +Other projects looking for a maintained library should use the upstream +univalue at https://github.com/jgarzik/univalue. diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 91b104e56e..6080516353 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -47,7 +47,6 @@ public: std::string s(val_); setStr(s); } - ~UniValue() {} void clear(); diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index eabcf2dad1..0ad6146545 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -35,7 +35,7 @@ bool ParseInt32(const std::string& str, int32_t *out) errno = 0; // strtol will not set errno if valid long int n = strtol(str.c_str(), &endp, 10); if(out) *out = (int32_t)n; - // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow + // Note that strtol returns a *long int*, so even if strtol doesn't report an over/underflow // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit // platforms the size of these types may be different. return endp && *endp == 0 && !errno && diff --git a/src/util/check.h b/src/util/check.h new file mode 100644 index 0000000000..d18887ae95 --- /dev/null +++ b/src/util/check.h @@ -0,0 +1,41 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_CHECK_H +#define BITCOIN_UTIL_CHECK_H + +#include <tinyformat.h> + +#include <stdexcept> + +class NonFatalCheckError : public std::runtime_error +{ + using std::runtime_error::runtime_error; +}; + +/** + * Throw a NonFatalCheckError when the condition evaluates to false + * + * This should only be used + * - where the condition is assumed to be true, not for error handling or validating user input + * - where a failure to fulfill the condition is recoverable and does not abort the program + * + * For example in RPC code, where it is undersirable to crash the whole program, this can be generally used to replace + * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC + * caller, which can then report the issue to the developers. + */ +#define CHECK_NONFATAL(condition) \ + do { \ + if (!(condition)) { \ + throw NonFatalCheckError( \ + strprintf("%s:%d (%s)\n" \ + "Internal bug detected: '%s'\n" \ + "You may report this issue here: %s\n", \ + __FILE__, __LINE__, __func__, \ + (#condition), \ + PACKAGE_BUGREPORT)); \ + } \ + } while (false) + +#endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/fees.cpp b/src/util/fees.cpp index cf16d5e44f..41149888d7 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -3,8 +3,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <util/fees.h> + #include <policy/fees.h> +#include <map> #include <string> std::string StringForFeeReason(FeeReason reason) { diff --git a/src/util/settings.cpp b/src/util/settings.cpp new file mode 100644 index 0000000000..af75fef310 --- /dev/null +++ b/src/util/settings.cpp @@ -0,0 +1,169 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/settings.h> + +#include <univalue.h> + +namespace util { +namespace { + +enum class Source { + FORCED, + COMMAND_LINE, + CONFIG_FILE_NETWORK_SECTION, + CONFIG_FILE_DEFAULT_SECTION +}; + +//! Merge settings from multiple sources in precedence order: +//! Forced config > command line > config file network-specific section > config file default section +//! +//! This function is provided with a callback function fn that contains +//! specific logic for how to merge the sources. +template <typename Fn> +static void MergeSettings(const Settings& settings, const std::string& section, const std::string& name, Fn&& fn) +{ + // Merge in the forced settings + if (auto* value = FindKey(settings.forced_settings, name)) { + fn(SettingsSpan(*value), Source::FORCED); + } + // Merge in the command-line options + if (auto* values = FindKey(settings.command_line_options, name)) { + fn(SettingsSpan(*values), Source::COMMAND_LINE); + } + // Merge in the network-specific section of the config file + if (!section.empty()) { + if (auto* map = FindKey(settings.ro_config, section)) { + if (auto* values = FindKey(*map, name)) { + fn(SettingsSpan(*values), Source::CONFIG_FILE_NETWORK_SECTION); + } + } + } + // Merge in the default section of the config file + if (auto* map = FindKey(settings.ro_config, "")) { + if (auto* values = FindKey(*map, name)) { + fn(SettingsSpan(*values), Source::CONFIG_FILE_DEFAULT_SECTION); + } + } +} +} // namespace + +SettingsValue GetSetting(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config, + bool get_chain_name) +{ + SettingsValue result; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + // Weird behavior preserved for backwards compatibility: Apply negated + // setting even if non-negated setting would be ignored. A negated + // value in the default section is applied to network specific options, + // even though normal non-negated values there would be ignored. + const bool never_ignore_negated_setting = span.last_negated(); + + // Weird behavior preserved for backwards compatibility: Take first + // assigned value instead of last. In general, later settings take + // precedence over early settings, but for backwards compatibility in + // the config file the precedence is reversed for all settings except + // chain name settings. + const bool reverse_precedence = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && !get_chain_name; + + // Weird behavior preserved for backwards compatibility: Negated + // -regtest and -testnet arguments which you would expect to override + // values set in the configuration file are currently accepted but + // silently ignored. It would be better to apply these just like other + // negated values, or at least warn they are ignored. + const bool skip_negated_command_line = get_chain_name; + + // Ignore settings in default config section if requested. + if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION && !never_ignore_negated_setting) return; + + // Skip negated command line settings. + if (skip_negated_command_line && span.last_negated()) return; + + // Stick with highest priority value, keeping result if already set. + if (!result.isNull()) return; + + if (!span.empty()) { + result = reverse_precedence ? span.begin()[0] : span.end()[-1]; + } else if (span.last_negated()) { + result = false; + } + }); + return result; +} + +std::vector<SettingsValue> GetSettingsList(const Settings& settings, + const std::string& section, + const std::string& name, + bool ignore_default_section_config) +{ + std::vector<SettingsValue> result; + bool result_complete = false; + bool prev_negated_empty = false; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + // Weird behavior preserved for backwards compatibility: Apply config + // file settings even if negated on command line. Negating a setting on + // command line will ignore earlier settings on the command line and + // ignore settings in the config file, unless the negated command line + // value is followed by non-negated value, in which case config file + // settings will be brought back from the dead (but earlier command + // line settings will still be ignored). + const bool add_zombie_config_values = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && !prev_negated_empty; + + // Ignore settings in default config section if requested. + if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION) return; + + // Add new settings to the result if isn't already complete, or if the + // values are zombies. + if (!result_complete || add_zombie_config_values) { + for (const auto& value : span) { + if (value.isArray()) { + result.insert(result.end(), value.getValues().begin(), value.getValues().end()); + } else { + result.push_back(value); + } + } + } + + // If a setting was negated, or if a setting was forced, set + // result_complete to true to ignore any later lower priority settings. + result_complete |= span.negated() > 0 || source == Source::FORCED; + + // Update the negated and empty state used for the zombie values check. + prev_negated_empty |= span.last_negated() && result.empty(); + }); + return result; +} + +bool OnlyHasDefaultSectionSetting(const Settings& settings, const std::string& section, const std::string& name) +{ + bool has_default_section_setting = false; + bool has_other_setting = false; + MergeSettings(settings, section, name, [&](SettingsSpan span, Source source) { + if (span.empty()) return; + else if (source == Source::CONFIG_FILE_DEFAULT_SECTION) has_default_section_setting = true; + else has_other_setting = true; + }); + // If a value is set in the default section and not explicitly overwritten by the + // user on the command line or in a different section, then we want to enable + // warnings about the value being ignored. + return has_default_section_setting && !has_other_setting; +} + +SettingsSpan::SettingsSpan(const std::vector<SettingsValue>& vec) noexcept : SettingsSpan(vec.data(), vec.size()) {} +const SettingsValue* SettingsSpan::begin() const { return data + negated(); } +const SettingsValue* SettingsSpan::end() const { return data + size; } +bool SettingsSpan::empty() const { return size == 0 || last_negated(); } +bool SettingsSpan::last_negated() const { return size > 0 && data[size - 1].isFalse(); } +size_t SettingsSpan::negated() const +{ + for (size_t i = size; i > 0; --i) { + if (data[i - 1].isFalse()) return i; // Return number of negated values (position of last false value) + } + return 0; +} + +} // namespace util diff --git a/src/util/settings.h b/src/util/settings.h new file mode 100644 index 0000000000..17832e4d2c --- /dev/null +++ b/src/util/settings.h @@ -0,0 +1,87 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SETTINGS_H +#define BITCOIN_UTIL_SETTINGS_H + +#include <map> +#include <string> +#include <vector> + +class UniValue; + +namespace util { + +//! Settings value type (string/integer/boolean/null variant). +//! +//! @note UniValue is used here for convenience and because it can be easily +//! serialized in a readable format. But any other variant type that can +//! be assigned strings, int64_t, and bool values and has get_str(), +//! get_int64(), get_bool(), isNum(), isBool(), isFalse(), isTrue() and +//! isNull() methods can be substituted if there's a need to move away +//! from UniValue. (An implementation with boost::variant was posted at +//! https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812) +using SettingsValue = UniValue; + +//! Stored bitcoin settings. This struct combines settings from the command line +//! and a read-only configuration file. +struct Settings { + //! Map of setting name to forced setting value. + std::map<std::string, SettingsValue> forced_settings; + //! Map of setting name to list of command line values. + std::map<std::string, std::vector<SettingsValue>> command_line_options; + //! Map of config section name and setting name to list of config file values. + std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config; +}; + +//! Get settings value from combined sources: forced settings, command line +//! arguments and the read-only config file. +//! +//! @param ignore_default_section_config - ignore values in the default section +//! of the config file (part before any +//! [section] keywords) +//! @param get_chain_name - enable special backwards compatible behavior +//! for GetChainName +SettingsValue GetSetting(const Settings& settings, const std::string& section, const std::string& name, bool ignore_default_section_config, bool get_chain_name); + +//! Get combined setting value similar to GetSetting(), except if setting was +//! specified multiple times, return a list of all the values specified. +std::vector<SettingsValue> GetSettingsList(const Settings& settings, const std::string& section, const std::string& name, bool ignore_default_section_config); + +//! Return true if a setting is set in the default config file section, and not +//! overridden by a higher priority command-line or network section value. +//! +//! This is used to provide user warnings about values that might be getting +//! ignored unintentionally. +bool OnlyHasDefaultSectionSetting(const Settings& settings, const std::string& section, const std::string& name); + +//! Accessor for list of settings that skips negated values when iterated over. +//! The last boolean `false` value in the list and all earlier values are +//! considered negated. +struct SettingsSpan { + explicit SettingsSpan() = default; + explicit SettingsSpan(const SettingsValue& value) noexcept : SettingsSpan(&value, 1) {} + explicit SettingsSpan(const SettingsValue* data, size_t size) noexcept : data(data), size(size) {} + explicit SettingsSpan(const std::vector<SettingsValue>& vec) noexcept; + const SettingsValue* begin() const; //<! Pointer to first non-negated value. + const SettingsValue* end() const; //<! Pointer to end of values. + bool empty() const; //<! True if there are any non-negated values. + bool last_negated() const; //<! True if the last value is negated. + size_t negated() const; //<! Number of negated values. + + const SettingsValue* data = nullptr; + size_t size = 0; +}; + +//! Map lookup helper. +template <typename Map, typename Key> +auto FindKey(Map&& map, Key&& key) -> decltype(&map.at(key)) +{ + auto it = map.find(key); + return it == map.end() ? nullptr : &it->second; +} + +} // namespace util + +#endif // BITCOIN_UTIL_SETTINGS_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 1e7d24c71c..46042f5634 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -138,7 +138,7 @@ std::string EncodeBase64(const unsigned char* pch, size_t len) std::string EncodeBase64(const std::string& str) { - return EncodeBase64((const unsigned char*)str.c_str(), str.size()); + return EncodeBase64((const unsigned char*)str.data(), str.size()); } std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) @@ -207,7 +207,7 @@ std::string EncodeBase32(const unsigned char* pch, size_t len) std::string EncodeBase32(const std::string& str) { - return EncodeBase32((const unsigned char*)str.c_str(), str.size()); + return EncodeBase32((const unsigned char*)str.data(), str.size()); } std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) diff --git a/src/util/system.cpp b/src/util/system.cpp index 526bf559c3..2a2ae6fdf5 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -63,6 +63,7 @@ #endif #include <thread> +#include <univalue.h> // Application startup time (used for uptime calculation) const int64_t nStartupTime = GetTime(); @@ -161,11 +162,14 @@ static bool InterpretBool(const std::string& strValue) return (atoi(strValue) != 0); } +static std::string SettingName(const std::string& arg) +{ + return arg.size() > 0 && arg[0] == '-' ? arg.substr(1) : arg; +} + /** Internal helper functions for ArgsManager */ class ArgsManagerHelper { public: - typedef std::map<std::string, std::vector<std::string>> MapArgs; - /** Determine whether to use config settings in the default section, * See also comments around ArgsManager::ArgsManager() below. */ static inline bool UseDefaultSection(const ArgsManager& am, const std::string& arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) @@ -173,91 +177,10 @@ public: return (am.m_network == CBaseChainParams::MAIN || am.m_network_only_args.count(arg) == 0); } - /** Convert regular argument into the network-specific setting */ - static inline std::string NetworkArg(const ArgsManager& am, const std::string& arg) - { - assert(arg.length() > 1 && arg[0] == '-'); - return "-" + am.m_network + "." + arg.substr(1); - } - - /** Find arguments in a map and add them to a vector */ - static inline void AddArgs(std::vector<std::string>& res, const MapArgs& map_args, const std::string& arg) - { - auto it = map_args.find(arg); - if (it != map_args.end()) { - res.insert(res.end(), it->second.begin(), it->second.end()); - } - } - - /** Return true/false if an argument is set in a map, and also - * return the first (or last) of the possibly multiple values it has - */ - static inline std::pair<bool,std::string> GetArgHelper(const MapArgs& map_args, const std::string& arg, bool getLast = false) - { - auto it = map_args.find(arg); - - if (it == map_args.end() || it->second.empty()) { - return std::make_pair(false, std::string()); - } - - if (getLast) { - return std::make_pair(true, it->second.back()); - } else { - return std::make_pair(true, it->second.front()); - } - } - - /* Get the string value of an argument, returning a pair of a boolean - * indicating the argument was found, and the value for the argument - * if it was found (or the empty string if not found). - */ - static inline std::pair<bool,std::string> GetArg(const ArgsManager &am, const std::string& arg) + static util::SettingsValue Get(const ArgsManager& am, const std::string& arg) { LOCK(am.cs_args); - std::pair<bool,std::string> found_result(false, std::string()); - - // We pass "true" to GetArgHelper in order to return the last - // argument value seen from the command line (so "bitcoind -foo=bar - // -foo=baz" gives GetArg(am,"foo")=={true,"baz"} - found_result = GetArgHelper(am.m_override_args, arg, true); - if (found_result.first) { - return found_result; - } - - // But in contrast we return the first argument seen in a config file, - // so "foo=bar \n foo=baz" in the config file gives - // GetArg(am,"foo")={true,"bar"} - if (!am.m_network.empty()) { - found_result = GetArgHelper(am.m_config_args, NetworkArg(am, arg)); - if (found_result.first) { - return found_result; - } - } - - if (UseDefaultSection(am, arg)) { - found_result = GetArgHelper(am.m_config_args, arg); - if (found_result.first) { - return found_result; - } - } - - return found_result; - } - - /* Special test for -testnet and -regtest args, because we - * don't want to be confused by craziness like "[regtest] testnet=1" - */ - static inline bool GetNetBoolArg(const ArgsManager &am, const std::string& net_arg) EXCLUSIVE_LOCKS_REQUIRED(am.cs_args) - { - std::pair<bool,std::string> found_result(false,std::string()); - found_result = GetArgHelper(am.m_override_args, net_arg, true); - if (!found_result.first) { - found_result = GetArgHelper(am.m_config_args, net_arg, true); - if (!found_result.first) { - return false; // not set - } - } - return InterpretBool(found_result.second); // is set, so evaluate + return GetSetting(am.m_settings, am.m_network, SettingName(arg), !UseDefaultSection(am, arg), /* get_chain_name= */ false); } }; @@ -268,13 +191,12 @@ public: * checks whether there was a double-negative (-nofoo=0 -> -foo=1). * * If there was not a double negative, it removes the "no" from the key - * and clears the args vector to indicate a negated option. + * and returns false. * - * If there was a double negative, it removes "no" from the key, sets the - * value to "1" and pushes the key and the updated value to the args vector. + * If there was a double negative, it removes "no" from the key, and + * returns true. * - * If there was no "no", it leaves key and value untouched and pushes them - * to the args vector. + * If there was no "no", it returns the string value untouched. * * Where an option was negated can be later checked using the * IsArgNegated() method. One use case for this is to have a way to disable @@ -282,34 +204,39 @@ public: * that debug log output is not sent to any file at all). */ -NODISCARD static bool InterpretOption(std::string key, std::string val, unsigned int flags, - std::map<std::string, std::vector<std::string>>& args, - std::string& error) +static util::SettingsValue InterpretOption(std::string& section, std::string& key, const std::string& value) { - assert(key[0] == '-'); - + // Split section name from key name for keys like "testnet.foo" or "regtest.bar" size_t option_index = key.find('.'); - if (option_index == std::string::npos) { - option_index = 1; - } else { - ++option_index; + if (option_index != std::string::npos) { + section = key.substr(0, option_index); + key.erase(0, option_index + 1); } - if (key.substr(option_index, 2) == "no") { - key.erase(option_index, 2); - if (flags & ArgsManager::ALLOW_BOOL) { - if (InterpretBool(val)) { - args[key].clear(); - return true; - } - // Double negatives like -nofoo=0 are supported (but discouraged) - LogPrintf("Warning: parsed potentially confusing double-negative %s=%s\n", key, val); - val = "1"; - } else { - error = strprintf("Negating of %s is meaningless and therefore forbidden", key.c_str()); - return false; + if (key.substr(0, 2) == "no") { + key.erase(0, 2); + // Double negatives like -nofoo=0 are supported (but discouraged) + if (!InterpretBool(value)) { + LogPrintf("Warning: parsed potentially confusing double-negative -%s=%s\n", key, value); + return true; } + return false; + } + return value; +} + +/** + * Check settings value validity according to flags. + * + * TODO: Add more meaningful error checks here in the future + * See "here's how the flags are meant to behave" in + * https://github.com/bitcoin/bitcoin/pull/16097#issuecomment-514627823 + */ +static bool CheckValid(const std::string& key, const util::SettingsValue& val, unsigned int flags, std::string& error) +{ + if (val.isBool() && !(flags & ArgsManager::ALLOW_BOOL)) { + error = strprintf("Negating of -%s is meaningless and therefore forbidden", key); + return false; } - args[key].push_back(val); return true; } @@ -331,22 +258,9 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const if (m_network == CBaseChainParams::MAIN) return std::set<std::string> {}; for (const auto& arg : m_network_only_args) { - std::pair<bool, std::string> found_result; - - // if this option is overridden it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_override_args, arg); - if (found_result.first) continue; - - // if there's a network-specific value for this option, it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_config_args, ArgsManagerHelper::NetworkArg(*this, arg)); - if (found_result.first) continue; - - // if there isn't a default value for this option, it's fine - found_result = ArgsManagerHelper::GetArgHelper(m_config_args, arg); - if (!found_result.first) continue; - - // otherwise, issue a warning - unsuitables.insert(arg); + if (OnlyHasDefaultSectionSetting(m_settings, m_network, SettingName(arg))) { + unsuitables.insert(arg); + } } return unsuitables; } @@ -375,7 +289,7 @@ void ArgsManager::SelectConfigNetwork(const std::string& network) bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::string& error) { LOCK(cs_args); - m_override_args.clear(); + m_settings.command_line_options.clear(); for (int i = 1; i < argc; i++) { std::string key(argv[i]); @@ -408,49 +322,44 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin if (key.length() > 1 && key[1] == '-') key.erase(0, 1); + // Transform -foo to foo + key.erase(0, 1); + std::string section; + util::SettingsValue value = InterpretOption(section, key, val); const unsigned int flags = FlagsOfKnownArg(key); if (flags) { - if (!InterpretOption(key, val, flags, m_override_args, error)) { + if (!CheckValid(key, value, flags, error)) { return false; } + // Weird behavior preserved for backwards compatibility: command + // line options with section prefixes are allowed but ignored. It + // would be better if these options triggered the Invalid parameter + // error below. + if (section.empty()) { + m_settings.command_line_options[key].push_back(value); + } } else { - error = strprintf("Invalid parameter %s", key.c_str()); + error = strprintf("Invalid parameter -%s", key); return false; } } // we do not allow -includeconf from command line, so we clear it here - auto it = m_override_args.find("-includeconf"); - if (it != m_override_args.end()) { - if (it->second.size() > 0) { - for (const auto& ic : it->second) { - error += "-includeconf cannot be used from commandline; -includeconf=" + ic + "\n"; - } - return false; + bool success = true; + if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + for (const auto& include : util::SettingsSpan(*includes)) { + error += "-includeconf cannot be used from commandline; -includeconf=" + include.get_str() + "\n"; + success = false; } } - return true; + return success; } unsigned int ArgsManager::FlagsOfKnownArg(const std::string& key) const { - assert(key[0] == '-'); - - size_t option_index = key.find('.'); - if (option_index == std::string::npos) { - option_index = 1; - } else { - ++option_index; - } - if (key.substr(option_index, 2) == "no") { - option_index += 2; - } - - const std::string base_arg_name = '-' + key.substr(option_index); - LOCK(cs_args); for (const auto& arg_map : m_available_args) { - const auto search = arg_map.second.find(base_arg_name); + const auto search = arg_map.second.find('-' + key); if (search != arg_map.second.end()) { return search->second.m_flags; } @@ -460,69 +369,42 @@ unsigned int ArgsManager::FlagsOfKnownArg(const std::string& key) const std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { - std::vector<std::string> result = {}; - if (IsArgNegated(strArg)) return result; // special case - LOCK(cs_args); - - ArgsManagerHelper::AddArgs(result, m_override_args, strArg); - if (!m_network.empty()) { - ArgsManagerHelper::AddArgs(result, m_config_args, ArgsManagerHelper::NetworkArg(*this, strArg)); + bool ignore_default_section_config = !ArgsManagerHelper::UseDefaultSection(*this, strArg); + std::vector<std::string> result; + for (const util::SettingsValue& value : + util::GetSettingsList(m_settings, m_network, SettingName(strArg), ignore_default_section_config)) { + result.push_back(value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str()); } - - if (ArgsManagerHelper::UseDefaultSection(*this, strArg)) { - ArgsManagerHelper::AddArgs(result, m_config_args, strArg); - } - return result; } bool ArgsManager::IsArgSet(const std::string& strArg) const { - if (IsArgNegated(strArg)) return true; // special case - return ArgsManagerHelper::GetArg(*this, strArg).first; + return !ArgsManagerHelper::Get(*this, strArg).isNull(); } bool ArgsManager::IsArgNegated(const std::string& strArg) const { - LOCK(cs_args); - - const auto& ov = m_override_args.find(strArg); - if (ov != m_override_args.end()) return ov->second.empty(); - - if (!m_network.empty()) { - const auto& cfs = m_config_args.find(ArgsManagerHelper::NetworkArg(*this, strArg)); - if (cfs != m_config_args.end()) return cfs->second.empty(); - } - - const auto& cf = m_config_args.find(strArg); - if (cf != m_config_args.end()) return cf->second.empty(); - - return false; + return ArgsManagerHelper::Get(*this, strArg).isFalse(); } std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) const { - if (IsArgNegated(strArg)) return "0"; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return found_res.second; - return strDefault; + const util::SettingsValue value = ArgsManagerHelper::Get(*this, strArg); + return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str(); } int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) const { - if (IsArgNegated(strArg)) return 0; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return atoi64(found_res.second); - return nDefault; + const util::SettingsValue value = ArgsManagerHelper::Get(*this, strArg); + return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : atoi64(value.get_str()); } bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const { - if (IsArgNegated(strArg)) return false; - std::pair<bool,std::string> found_res = ArgsManagerHelper::GetArg(*this, strArg); - if (found_res.first) return InterpretBool(found_res.second); - return fDefault; + const util::SettingsValue value = ArgsManagerHelper::Get(*this, strArg); + return value.isNull() ? fDefault : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); } bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) @@ -544,7 +426,7 @@ bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); - m_override_args[strArg] = {strValue}; + m_settings.forced_settings[SettingName(strArg)] = strValue; } void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) @@ -688,7 +570,7 @@ void PrintExceptionContinue(const std::exception* pex, const char* pszThread) { std::string message = FormatException(pex, pszThread); LogPrintf("\n\n************************\n%s\n", message); - tfm::format(std::cerr, "\n\n************************\n%s\n", message.c_str()); + tfm::format(std::cerr, "\n\n************************\n%s\n", message); } fs::path GetDefaultDataDir() @@ -860,17 +742,20 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file return false; } for (const std::pair<std::string, std::string>& option : options) { - const std::string strKey = std::string("-") + option.first; - const unsigned int flags = FlagsOfKnownArg(strKey); + std::string section; + std::string key = option.first; + util::SettingsValue value = InterpretOption(section, key, option.second); + const unsigned int flags = FlagsOfKnownArg(key); if (flags) { - if (!InterpretOption(strKey, option.second, flags, m_config_args, error)) { + if (!CheckValid(key, value, flags, error)) { return false; } + m_settings.ro_config[section][key].push_back(value); } else { if (ignore_invalid_keys) { LogPrintf("Ignoring unknown configuration value %s\n", option.first); } else { - error = strprintf("Invalid configuration value %s", option.first.c_str()); + error = strprintf("Invalid configuration value %s", option.first); return false; } } @@ -882,7 +767,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) { { LOCK(cs_args); - m_config_args.clear(); + m_settings.ro_config.clear(); m_config_sections.clear(); } @@ -894,58 +779,64 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { return false; } - // if there is an -includeconf in the override args, but it is empty, that means the user - // passed '-noincludeconf' on the command line, in which case we should not include anything - bool emptyIncludeConf; + // `-includeconf` cannot be included in the command line arguments except + // as `-noincludeconf` (which indicates that no conf file should be used). + bool use_conf_file{true}; { LOCK(cs_args); - emptyIncludeConf = m_override_args.count("-includeconf") == 0; + if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + // ParseParameters() fails if a non-negated -includeconf is passed on the command-line + assert(util::SettingsSpan(*includes).last_negated()); + use_conf_file = false; + } } - if (emptyIncludeConf) { + if (use_conf_file) { std::string chain_id = GetChainName(); - std::vector<std::string> includeconf(GetArgs("-includeconf")); - { - // We haven't set m_network yet (that happens in SelectParams()), so manually check - // for network.includeconf args. - std::vector<std::string> includeconf_net(GetArgs(std::string("-") + chain_id + ".includeconf")); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - } + std::vector<std::string> conf_file_names; - // Remove -includeconf from configuration, so we can warn about recursion - // later - { + auto add_includes = [&](const std::string& network, size_t skip = 0) { + size_t num_values = 0; LOCK(cs_args); - m_config_args.erase("-includeconf"); - m_config_args.erase(std::string("-") + chain_id + ".includeconf"); - } - - for (const std::string& to_include : includeconf) { - fsbridge::ifstream include_config(GetConfigFile(to_include)); - if (include_config.good()) { - if (!ReadConfigStream(include_config, to_include, error, ignore_invalid_keys)) { + if (auto* section = util::FindKey(m_settings.ro_config, network)) { + if (auto* values = util::FindKey(*section, "includeconf")) { + for (size_t i = std::max(skip, util::SettingsSpan(*values).negated()); i < values->size(); ++i) { + conf_file_names.push_back((*values)[i].get_str()); + } + num_values = values->size(); + } + } + return num_values; + }; + + // We haven't set m_network yet (that happens in SelectParams()), so manually check + // for network.includeconf args. + const size_t chain_includes = add_includes(chain_id); + const size_t default_includes = add_includes({}); + + for (const std::string& conf_file_name : conf_file_names) { + fsbridge::ifstream conf_file_stream(GetConfigFile(conf_file_name)); + if (conf_file_stream.good()) { + if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; } - LogPrintf("Included configuration file %s\n", to_include.c_str()); + LogPrintf("Included configuration file %s\n", conf_file_name); } else { - error = "Failed to include configuration file " + to_include; + error = "Failed to include configuration file " + conf_file_name; return false; } } // Warn about recursive -includeconf - includeconf = GetArgs("-includeconf"); - { - std::vector<std::string> includeconf_net(GetArgs(std::string("-") + chain_id + ".includeconf")); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - std::string chain_id_final = GetChainName(); - if (chain_id_final != chain_id) { - // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs - includeconf_net = GetArgs(std::string("-") + chain_id_final + ".includeconf"); - includeconf.insert(includeconf.end(), includeconf_net.begin(), includeconf_net.end()); - } + conf_file_names.clear(); + add_includes(chain_id, /* skip= */ chain_includes); + add_includes({}, /* skip= */ default_includes); + std::string chain_id_final = GetChainName(); + if (chain_id_final != chain_id) { + // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs + add_includes(chain_id_final); } - for (const std::string& to_include : includeconf) { - tfm::format(std::cerr, "warning: -includeconf cannot be used from included files; ignoring -includeconf=%s\n", to_include.c_str()); + for (const std::string& conf_file_name : conf_file_names) { + tfm::format(std::cerr, "warning: -includeconf cannot be used from included files; ignoring -includeconf=%s\n", conf_file_name); } } } @@ -953,7 +844,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) // If datadir is changed in .conf file: ClearDatadirCache(); if (!CheckDataDirOption()) { - error = strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "").c_str()); + error = strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "")); return false; } return true; @@ -961,9 +852,16 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) std::string ArgsManager::GetChainName() const { - LOCK(cs_args); - const bool fRegTest = ArgsManagerHelper::GetNetBoolArg(*this, "-regtest"); - const bool fTestNet = ArgsManagerHelper::GetNetBoolArg(*this, "-testnet"); + auto get_net = [&](const std::string& arg) { + LOCK(cs_args); + util::SettingsValue value = GetSetting(m_settings, /* section= */ "", SettingName(arg), + /* ignore_default_section_config= */ false, + /* get_chain_name= */ true); + return value.isNull() ? false : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); + }; + + const bool fRegTest = get_net("-regtest"); + const bool fTestNet = get_net("-testnet"); const bool is_chain_arg_set = IsArgSet("-chain"); if ((int)is_chain_arg_set + (int)fRegTest + (int)fTestNet > 1) { diff --git a/src/util/system.h b/src/util/system.h index 908a3c407d..e0b6371dc9 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -22,6 +22,7 @@ #include <sync.h> #include <tinyformat.h> #include <util/memory.h> +#include <util/settings.h> #include <util/threadnames.h> #include <util/time.h> @@ -157,8 +158,7 @@ protected: }; mutable CCriticalSection cs_args; - std::map<std::string, std::vector<std::string>> m_override_args GUARDED_BY(cs_args); - std::map<std::string, std::vector<std::string>> m_config_args GUARDED_BY(cs_args); + util::Settings m_settings GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); @@ -265,7 +265,7 @@ public: void ForceSetArg(const std::string& strArg, const std::string& strValue); /** - * Looks for -regtest, -testnet and returns the appropriate BIP70 chain name. + * Returns the appropriate chain name from the program arguments. * @return CBaseChainParams::MAIN by default; raises runtime error if an invalid combination is given. */ std::string GetChainName() const; diff --git a/src/util/time.cpp b/src/util/time.cpp index 2b202ae95f..2afff2626b 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -111,3 +111,17 @@ std::string FormatISO8601Date(int64_t nTime) { #endif return strprintf("%04i-%02i-%02i", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday); } + +int64_t ParseISO8601DateTime(const std::string& str) +{ + static const boost::posix_time::ptime epoch = boost::posix_time::from_time_t(0); + static const std::locale loc(std::locale::classic(), + new boost::posix_time::time_input_facet("%Y-%m-%dT%H:%M:%SZ")); + std::istringstream iss(str); + iss.imbue(loc); + boost::posix_time::ptime ptime(boost::date_time::not_a_date_time); + iss >> ptime; + if (ptime.is_not_a_date_time() || epoch > ptime) + return 0; + return (ptime - epoch).total_seconds(); +}
\ No newline at end of file diff --git a/src/util/time.h b/src/util/time.h index c0470a2136..af4390aa1c 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -48,5 +48,6 @@ T GetTime(); */ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); +int64_t ParseISO8601DateTime(const std::string& str); #endif // BITCOIN_UTIL_TIME_H diff --git a/src/util/translation.h b/src/util/translation.h index 0e6eb5a094..fc45da440a 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -6,7 +6,7 @@ #define BITCOIN_UTIL_TRANSLATION_H #include <tinyformat.h> - +#include <functional> /** * Bilingual messages: diff --git a/src/util/validation.cpp b/src/util/validation.cpp index fe1f5a277e..bd52f57751 100644 --- a/src/util/validation.cpp +++ b/src/util/validation.cpp @@ -8,13 +8,12 @@ #include <consensus/validation.h> #include <tinyformat.h> -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state) +/** Convert ValidationState to a human-readable message for logging */ +std::string FormatStateMessage(const ValidationState &state) { - return strprintf("%s%s (code %i)", + return strprintf("%s%s", state.GetRejectReason(), - state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage(), - state.GetRejectCode()); + state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage()); } const std::string strMessageMagic = "Bitcoin Signed Message:\n"; diff --git a/src/util/validation.h b/src/util/validation.h index 32559853ee..da2cf9f102 100644 --- a/src/util/validation.h +++ b/src/util/validation.h @@ -8,10 +8,10 @@ #include <string> -class CValidationState; +class ValidationState; -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state); +/** Convert ValidationState to a human-readable message for logging */ +std::string FormatStateMessage(const ValidationState &state); extern const std::string strMessageMagic; diff --git a/src/validation.cpp b/src/validation.cpp index 7db1ade315..e744110371 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -18,6 +18,8 @@ #include <flatfile.h> #include <hash.h> #include <index/txindex.h> +#include <logging.h> +#include <logging/timer.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -107,7 +109,7 @@ CBlockIndex *pindexBestHeader = nullptr; Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; -int nScriptCheckThreads = 0; +bool g_parallel_script_checks{false}; std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); bool fHavePruned = false; @@ -181,7 +183,7 @@ std::unique_ptr<CBlockTreeDB> pblocktree; // See definition for documentation static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); +bool CheckInputs(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); @@ -363,9 +365,9 @@ static void UpdateMempoolForReorg(DisconnectedBlockTransactions& disconnectpool, auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions - CValidationState stateDummy; + TxValidationState stateDummy; if (!fAddToMempool || (*it)->IsCoinBase() || - !AcceptToMemoryPool(mempool, stateDummy, *it, nullptr /* pfMissingInputs */, + !AcceptToMemoryPool(mempool, stateDummy, *it, nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */)) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). @@ -391,8 +393,8 @@ static void UpdateMempoolForReorg(DisconnectedBlockTransactions& disconnectpool, // Used to avoid mempool polluting consensus critical paths if CCoinsViewMempool // were somehow broken and returning the wrong scriptPubKeys -static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, - unsigned int flags, bool cacheSigStore, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { +static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, + unsigned int flags, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); // pool.cs should be locked already, but go ahead and re-take the lock here @@ -422,7 +424,8 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, CValidationSt } } - return CheckInputs(tx, state, view, flags, cacheSigStore, true, txdata); + // Call CheckInputs() to cache signature and script validity against current tip consensus rules. + return CheckInputs(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); } namespace { @@ -440,8 +443,7 @@ public: // around easier. struct ATMPArgs { const CChainParams& m_chainparams; - CValidationState &m_state; - bool* m_missing_inputs; + TxValidationState &m_state; const int64_t m_accept_time; std::list<CTransactionRef>* m_replaced_transactions; const bool m_bypass_limits; @@ -501,15 +503,15 @@ private: bool Finalize(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Compare a package's feerate against minimum allowed. - bool CheckFeeRate(size_t package_size, CAmount package_fee, CValidationState& state) + bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) { CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } if (package_fee < ::minRelayTxFee.GetFee(package_size)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "min relay fee not met", strprintf("%d < %d", package_fee, ::minRelayTxFee.GetFee(package_size))); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", package_fee, ::minRelayTxFee.GetFee(package_size))); } return true; } @@ -536,8 +538,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const uint256& hash = ws.m_hash; // Copy/alias what we need out of args - CValidationState &state = args.m_state; - bool* pfMissingInputs = args.m_missing_inputs; + TxValidationState &state = args.m_state; const int64_t nAcceptTime = args.m_accept_time; const bool bypass_limits = args.m_bypass_limits; const CAmount& nAbsurdFee = args.m_absurd_fee; @@ -553,38 +554,34 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount& nConflictingFees = ws.m_conflicting_fees; size_t& nConflictingSize = ws.m_conflicting_size; - if (pfMissingInputs) { - *pfMissingInputs = false; - } - if (!CheckTransaction(tx, state)) return false; // state filled in by CheckTransaction // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "coinbase"); + return state.Invalid(TxValidationResult::TX_CONSENSUS, "coinbase"); // Rather not work on nonstandard transactions (unless -testnet/-regtest) std::string reason; if (fRequireStandard && !IsStandardTx(tx, reason)) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, reason); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, reason); // Do not work on transactions that are too small. // A transaction with 1 segwit input and 1 P2WPHK output has non-witness size of 82 bytes. // Transactions smaller than this are not relayed to mitigate CVE-2017-12842 by not relaying // 64-byte transactions. if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, "tx-size-small"); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) - return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, REJECT_NONSTANDARD, "non-final"); + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); // is it already in the memory pool? if (m_pool.exists(hash)) { - return state.Invalid(ValidationInvalidReason::TX_CONFLICT, false, REJECT_DUPLICATE, "txn-already-in-mempool"); + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); } // Check for conflicts with in-memory transactions @@ -616,7 +613,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } if (fReplacementOptOut) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_DUPLICATE, "txn-mempool-conflict"); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } setConflicts.insert(ptxConflicting->GetHash()); @@ -642,14 +639,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) for (size_t out = 0; out < tx.vout.size(); out++) { // Optimistically just do efficient check of cache for outputs if (coins_cache.HaveCoinInCache(COutPoint(hash, out))) { - return state.Invalid(ValidationInvalidReason::TX_CONFLICT, false, REJECT_DUPLICATE, "txn-already-known"); + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-known"); } } // Otherwise assume this might be an orphan tx for which we just haven't seen parents yet - if (pfMissingInputs) { - *pfMissingInputs = true; - } - return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() + return state.Invalid(TxValidationResult::TX_MISSING_INPUTS, "bad-txns-inputs-missingorspent"); } } @@ -667,7 +661,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Must keep pool.cs for this unless we change CheckSequenceLocks to take a // CoinsViewCache instead of create its own if (!CheckSequenceLocks(m_pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) - return state.Invalid(ValidationInvalidReason::TX_PREMATURE_SPEND, false, REJECT_NONSTANDARD, "non-BIP68-final"); + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); CAmount nFees = 0; if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { @@ -676,11 +670,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Check for non-standard pay-to-script-hash in inputs if (fRequireStandard && !AreInputsStandard(tx, m_view)) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-nonstandard-inputs"); // Check for non-standard witness in P2WSH if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, m_view)) - return state.Invalid(ValidationInvalidReason::TX_WITNESS_MUTATED, false, REJECT_NONSTANDARD, "bad-witness-nonstandard"); + return state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, "bad-witness-nonstandard"); int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); @@ -704,7 +698,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) unsigned int nSize = entry->GetTxSize(); if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", strprintf("%d", nSigOpsCost)); // No transactions are allowed below minRelayTxFee except from disconnected @@ -712,9 +706,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!bypass_limits && !CheckFeeRate(nSize, nModifiedFees, state)) return false; if (nAbsurdFee && nFees > nAbsurdFee) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, - REJECT_HIGHFEE, "absurdly-high-fee", - strprintf("%d > %d", nFees, nAbsurdFee)); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, + "absurdly-high-fee", strprintf("%d > %d", nFees, nAbsurdFee)); const CTxMemPool::setEntries setIterConflicting = m_pool.GetIterSet(setConflicts); // Calculate in-mempool ancestors, up to a limit. @@ -742,7 +735,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // To check these we first check if we meet the RBF criteria, above, and increment the descendant // limits by the direct conflict and its descendants (as these are recalculated in // CalculateMempoolAncestors by assuming the new transaction being added is a new descendant, with no - // removals, of each parent's existing dependant set). The ancestor count limits are unmodified (as + // removals, of each parent's existing dependent set). The ancestor count limits are unmodified (as // the ancestor limits should be the same for both our new transaction and any conflicts). // We don't bother incrementing m_limit_descendants by the full removal count as that limit never comes // into force here (as we're only adding a single transaction). @@ -771,7 +764,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // this, see https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html if (nSize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || !m_pool.CalculateMemPoolAncestors(*entry, setAncestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_NONSTANDARD, "too-long-mempool-chain", errString); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", errString); } } @@ -784,7 +777,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); if (setConflicts.count(hashAncestor)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-spends-conflicting-tx", + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", strprintf("%s spends conflicting transaction %s", hash.ToString(), hashAncestor.ToString())); @@ -824,7 +817,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); if (newFeeRate <= oldFeeRate) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "insufficient fee", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", hash.ToString(), newFeeRate.ToString(), @@ -852,7 +845,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) nConflictingSize += it->GetTxSize(); } } else { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_NONSTANDARD, "too many potential replacements", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too many potential replacements", strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", hash.ToString(), nConflictingCount, @@ -876,7 +869,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // it's cheaper to just check if the new input refers to a // tx that's in the mempool. if (m_pool.exists(tx.vin[j].prevout.hash)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_NONSTANDARD, "replacement-adds-unconfirmed", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "replacement-adds-unconfirmed", strprintf("replacement %s adds unconfirmed input, idx %d", hash.ToString(), j)); } @@ -888,7 +881,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // transactions would not be paid for. if (nModifiedFees < nConflictingFees) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "insufficient fee", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); } @@ -898,7 +891,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CAmount nDeltaFees = nModifiedFees - nConflictingFees; if (nDeltaFees < ::incrementalRelayFee.GetFee(nSize)) { - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "insufficient fee", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", hash.ToString(), FormatMoney(nDeltaFees), @@ -912,7 +905,7 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, Precompute { const CTransaction& tx = *ws.m_ptx; - CValidationState &state = args.m_state; + TxValidationState &state = args.m_state; constexpr unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; @@ -922,14 +915,13 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, Precompute // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we // need to turn both off, and compare against just turning off CLEANSTACK // to see if the failure is specifically due to witness validation. - CValidationState stateDummy; // Want reported failures to be from first CheckInputs - if (!tx.HasWitness() && CheckInputs(tx, stateDummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && - !CheckInputs(tx, stateDummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { + TxValidationState state_dummy; // Want reported failures to be from first CheckInputs + if (!tx.HasWitness() && CheckInputs(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && + !CheckInputs(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { // Only the witness is missing, so the transaction itself may be fine. - state.Invalid(ValidationInvalidReason::TX_WITNESS_MUTATED, false, - state.GetRejectCode(), state.GetRejectReason(), state.GetDebugMessage()); + state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, + state.GetRejectReason(), state.GetDebugMessage()); } - assert(IsTransactionReason(state.GetReason())); return false; // state filled in by CheckInputs } @@ -941,7 +933,7 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, Precomp const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - CValidationState &state = args.m_state; + TxValidationState &state = args.m_state; const CChainParams& chainparams = args.m_chainparams; // Check again against the current block tip's script verification @@ -960,7 +952,7 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, Precomp // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(::ChainActive().Tip(), chainparams.GetConsensus()); - if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, true, txdata)) { + if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata)) { return error("%s: BUG! PLEASE REPORT THIS! CheckInputs failed against latest-block but not STANDARD flags %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } @@ -972,7 +964,7 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; - CValidationState &state = args.m_state; + TxValidationState &state = args.m_state; const bool bypass_limits = args.m_bypass_limits; CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; @@ -1010,7 +1002,7 @@ bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) if (!bypass_limits) { LimitMempoolSize(m_pool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); if (!m_pool.exists(hash)) - return state.Invalid(ValidationInvalidReason::TX_MEMPOOL_POLICY, false, REJECT_INSUFFICIENTFEE, "mempool full"); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } return true; } @@ -1047,12 +1039,12 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs } // anon namespace /** (try to) add transaction to memory pool with a specified acceptance time **/ -static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, - bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, +static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, state, pfMissingInputs, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept }; + MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept }; bool res = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); if (!res) { // Remove coins that were not present in the coins cache before calling ATMPW; @@ -1064,17 +1056,17 @@ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPo ::ChainstateActive().CoinsTip().Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits - CValidationState stateDummy; - ::ChainstateActive().FlushStateToDisk(chainparams, stateDummy, FlushStateMode::PERIODIC); + BlockValidationState state_dummy; + ::ChainstateActive().FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); return res; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, - bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced, +bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) { const CChainParams& chainparams = Params(); - return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, pfMissingInputs, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); + return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); } /** @@ -1419,8 +1411,8 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c CheckForkWarningConditions(); } -void CChainState::InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { - if (state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { +void CChainState::InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) { + if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; m_blockman.m_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); @@ -1493,7 +1485,7 @@ void InitScriptExecutionCache() { * * Non-static (and re-declared) in src/test/txvalidationcache_tests.cpp */ -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool CheckInputs(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (tx.IsCoinBase()) return true; @@ -1545,10 +1537,10 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi CScriptCheck check2(coin.out, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheSigStore, &txdata); if (check2()) - return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } // MANDATORY flag failures correspond to - // ValidationInvalidReason::CONSENSUS. Because CONSENSUS + // TxValidationResult::TX_CONSENSUS. Because CONSENSUS // failures are the most serious case of validation // failures, we may need to consider using // RECENT_CONSENSUS_CHANGE for any script failure that @@ -1556,7 +1548,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // support, to avoid splitting the network (but this // depends on the details of how net_processing handles // such errors). - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); + return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } @@ -1641,7 +1633,7 @@ static bool AbortNode(const std::string& strMessage, const std::string& userMess return false; } -static bool AbortNode(CValidationState& state, const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) +static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) { AbortNode(strMessage, userMessage, prefix); return state.Error(strMessage); @@ -1755,9 +1747,9 @@ void static FlushBlockFile(bool fFinalize = false) } } -static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); +static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); -static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { @@ -1896,7 +1888,7 @@ static int64_t nBlocksTotal = 0; /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ -bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, +bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { AssertLockHeld(cs_main); @@ -1918,7 +1910,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // re-enforce that rule here (at least until we make it impossible for // GetAdjustedTime() to go backward). if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) { - if (state.GetReason() == ValidationInvalidReason::BLOCK_MUTATED) { + if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) { // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware // problems. @@ -2058,8 +2050,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl for (const auto& tx : block.vtx) { for (size_t o = 0; o < tx->vout.size(); o++) { if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + LogPrintf("ERROR: ConnectBlock(): tried to overwrite transaction\n"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-BIP30"); } } } @@ -2079,7 +2071,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl CBlockUndo blockundo; - CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : nullptr); + CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); std::vector<int> prevheights; CAmount nFees = 0; @@ -2097,21 +2089,17 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl if (!tx.IsCoinBase()) { CAmount txfee = 0; - if (!Consensus::CheckTxInputs(tx, state, view, pindex->nHeight, txfee)) { - if (!IsBlockReason(state.GetReason())) { - // CheckTxInputs may return MISSING_INPUTS or - // PREMATURE_SPEND but we can't return that, as it's not - // defined for a block, so we reset the reason flag to - // CONSENSUS here. - state.Invalid(ValidationInvalidReason::CONSENSUS, false, - state.GetRejectCode(), state.GetRejectReason(), state.GetDebugMessage()); - } + TxValidationState tx_state; + if (!Consensus::CheckTxInputs(tx, tx_state, view, pindex->nHeight, txfee)) { + // Any transaction validation failure in ConnectBlock is a block consensus failure + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, + tx_state.GetRejectReason(), tx_state.GetDebugMessage()); return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } nFees += txfee; if (!MoneyRange(nFees)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: accumulated fee in the block out of range.", __func__), - REJECT_INVALID, "bad-txns-accumulated-fee-outofrange"); + LogPrintf("ERROR: %s: accumulated fee in the block out of range.\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-accumulated-fee-outofrange"); } // Check that transaction is BIP68 final @@ -2123,8 +2111,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: contains a non-BIP68-final transaction", __func__), - REJECT_INVALID, "bad-txns-nonfinal"); + LogPrintf("ERROR: %s: contains a non-BIP68-final transaction\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal"); } } @@ -2133,26 +2121,21 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl // * p2sh (when P2SH enabled in flags and excludes coinbase) // * witness (when witness enabled in flags and excludes coinbase) nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); - if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("ConnectBlock(): too many sigops"), - REJECT_INVALID, "bad-blk-sigops"); + if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) { + LogPrintf("ERROR: ConnectBlock(): too many sigops\n"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); + } txdata.emplace_back(tx); if (!tx.IsCoinBase()) { std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ - if (fScriptChecks && !CheckInputs(tx, state, view, flags, fCacheResults, fCacheResults, txdata[i], nScriptCheckThreads ? &vChecks : nullptr)) { - if (state.GetReason() == ValidationInvalidReason::TX_NOT_STANDARD) { - // CheckInputs may return NOT_STANDARD for extra flags we passed, - // but we can't return that, as it's not defined for a block, so - // we reset the reason flag to CONSENSUS here. - // In the event of a future soft-fork, we may need to - // consider whether rewriting to CONSENSUS or - // RECENT_CONSENSUS_CHANGE would be more appropriate. - state.Invalid(ValidationInvalidReason::CONSENSUS, false, - state.GetRejectCode(), state.GetRejectReason(), state.GetDebugMessage()); - } + TxValidationState tx_state; + if (fScriptChecks && !CheckInputs(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + // Any transaction validation failure in ConnectBlock is a block consensus failure + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, + tx_state.GetRejectReason(), tx_state.GetDebugMessage()); return error("ConnectBlock(): CheckInputs on %s failed with %s", tx.GetHash().ToString(), FormatStateMessage(state)); } @@ -2169,14 +2152,15 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); - if (block.vtx[0]->GetValueOut() > blockReward) - return state.Invalid(ValidationInvalidReason::CONSENSUS, - error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", - block.vtx[0]->GetValueOut(), blockReward), - REJECT_INVALID, "bad-cb-amount"); - - if (!control.Wait()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); + if (block.vtx[0]->GetValueOut() > blockReward) { + LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); + } + + if (!control.Wait()) { + LogPrintf("ERROR: %s: CheckQueue failed\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "block-validation-failed"); + } int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); @@ -2206,7 +2190,7 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl bool CChainState::FlushStateToDisk( const CChainParams& chainparams, - CValidationState &state, + BlockValidationState &state, FlushStateMode mode, int nManualPruneHeight) { @@ -2217,6 +2201,10 @@ bool CChainState::FlushStateToDisk( static int64_t nLastFlush = 0; std::set<int> setFilesToPrune; bool full_flush_completed = false; + + const size_t coins_count = CoinsTip().GetCacheSize(); + const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage(); + try { { bool fFlushForPrune = false; @@ -2224,8 +2212,12 @@ bool CChainState::FlushStateToDisk( LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { + LOG_TIME_MILLIS("find files to prune (manual)", BCLog::BENCH); + FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); } else { + LOG_TIME_MILLIS("find files to prune", BCLog::BENCH); + FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); fCheckForPruning = false; } @@ -2264,10 +2256,17 @@ bool CChainState::FlushStateToDisk( if (!CheckDiskSpace(GetBlocksDir())) { return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } - // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); + { + LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCH); + + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + } + // Then update all block file information (which may refer to block and undo files). { + LOG_TIME_MILLIS("write block index to disk", BCLog::BENCH); + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; vFiles.reserve(setDirtyFileInfo.size()); for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { @@ -2285,12 +2284,18 @@ bool CChainState::FlushStateToDisk( } } // Finally remove any pruned files - if (fFlushForPrune) + if (fFlushForPrune) { + LOG_TIME_MILLIS("unlink pruned files", BCLog::BENCH); + UnlinkPrunedFiles(setFilesToPrune); + } nLastWrite = nNow; } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { + LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000)); + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already @@ -2317,7 +2322,7 @@ bool CChainState::FlushStateToDisk( } void CChainState::ForceFlushStateToDisk() { - CValidationState state; + BlockValidationState state; const CChainParams& chainparams = Params(); if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, FormatStateMessage(state)); @@ -2325,7 +2330,7 @@ void CChainState::ForceFlushStateToDisk() { } void CChainState::PruneAndFlush() { - CValidationState state; + BlockValidationState state; fCheckForPruning = true; const CChainParams& chainparams = Params(); @@ -2411,7 +2416,7 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ -bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) +bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); @@ -2453,7 +2458,7 @@ bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& cha UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - GetMainSignals().BlockDisconnected(pblock); + GetMainSignals().BlockDisconnected(pblock, pindexDelete); return true; } @@ -2531,7 +2536,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) +bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. @@ -2663,7 +2668,7 @@ void CChainState::PruneBlockIndexCandidates() { * * @returns true unless a system error occurred */ -bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); @@ -2710,10 +2715,10 @@ bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainPar if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. - if (state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { + if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { InvalidChainFound(vpindexToConnect.front()); } - state = CValidationState(); + state = BlockValidationState(); fInvalidFound = true; fContinue = false; break; @@ -2781,7 +2786,7 @@ static void LimitValidationInterfaceQueue() LOCKS_EXCLUDED(cs_main) { } } -bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set @@ -2881,11 +2886,11 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& return true; } -bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { return ::ChainstateActive().ActivateBestChain(state, chainparams, std::move(pblock)); } -bool CChainState::PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) +bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { LOCK(cs_main); @@ -2913,11 +2918,11 @@ bool CChainState::PreciousBlock(CValidationState& state, const CChainParams& par return ActivateBestChain(state, params, std::shared_ptr<const CBlock>()); } -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) { +bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { return ::ChainstateActive().PreciousBlock(state, params, pindex); } -bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { CBlockIndex* to_mark_failed = pindex; bool pindex_was_in_chain = false; @@ -3053,7 +3058,7 @@ bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& c return true; } -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { +bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { return ::ChainstateActive().InvalidateBlock(state, chainparams, pindex); } @@ -3227,7 +3232,7 @@ static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int n return true; } -static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) +static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; @@ -3249,16 +3254,16 @@ static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, un return true; } -static bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) +static bool CheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_INVALID, "high-hash", "proof of work failed"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "high-hash", "proof of work failed"); return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. @@ -3275,13 +3280,13 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txnmrklroot", "hashMerkleRoot mismatch"); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-txnmrklroot", "hashMerkleRoot mismatch"); // Check for merkle tree malleability (CVE-2012-2459): repeating sequences // of transactions in a block without affecting the merkle root of a block, // while still invalidating it. if (mutated) - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-txns-duplicate", "duplicate transaction"); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-txns-duplicate", "duplicate transaction"); } // All potential-corruption validation must be done before we do any @@ -3292,29 +3297,34 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P // Size limits if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-blk-length", "size limits failed"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-missing", "first tx is not coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-missing", "first tx is not coinbase"); for (unsigned int i = 1; i < block.vtx.size(); i++) if (block.vtx[i]->IsCoinBase()) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-multiple", "more than one coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-multiple", "more than one coinbase"); // Check transactions // Must check for duplicate inputs (see CVE-2018-17144) - for (const auto& tx : block.vtx) - if (!CheckTransaction(*tx, state, true)) - return state.Invalid(state.GetReason(), false, state.GetRejectCode(), state.GetRejectReason(), - strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), state.GetDebugMessage())); - + for (const auto& tx : block.vtx) { + TxValidationState tx_state; + if (!CheckTransaction(*tx, tx_state)) { + // CheckBlock() does context-free validation checks. The only + // possible failures are consensus failures. + assert(tx_state.GetResult() == TxValidationResult::TX_CONSENSUS); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), + strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), tx_state.GetDebugMessage())); + } + } unsigned int nSigOps = 0; for (const auto& tx : block.vtx) { nSigOps += GetLegacySigOpCount(*tx); } if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-blk-sigops", "out-of-bounds SigOpCount"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops", "out-of-bounds SigOpCount"); if (fCheckPOW && fCheckMerkleRoot) block.fChecked = true; @@ -3407,7 +3417,7 @@ static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOC * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { assert(pindexPrev != nullptr); const int nHeight = pindexPrev->nHeight + 1; @@ -3415,7 +3425,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta // Check proof of work const Consensus::Params& consensusParams = params.GetConsensus(); if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_INVALID, "bad-diffbits", "incorrect proof of work"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "bad-diffbits", "incorrect proof of work"); // Check against checkpoints if (fCheckpointsEnabled) { @@ -3423,24 +3433,26 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // g_blockman.m_block_index. CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.Invalid(ValidationInvalidReason::BLOCK_CHECKPOINT, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); + if (pcheckpoint && nHeight < pcheckpoint->nHeight) { + LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); + return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); + } } // Check timestamp against prev if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "time-too-old", "block's timestamp is too early"); // Check timestamp if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) - return state.Invalid(ValidationInvalidReason::BLOCK_TIME_FUTURE, false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); + return state.Invalid(BlockValidationResult::BLOCK_TIME_FUTURE, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: // check for version 2, 3 and 4 upgrades if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_HEADER, false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, strprintf("bad-version(0x%08x)", block.nVersion), strprintf("rejected nVersion=0x%08x block", block.nVersion)); return true; @@ -3452,7 +3464,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) +static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; @@ -3470,7 +3482,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // Check that all transactions are finalized for (const auto& tx : block.vtx) { if (!IsFinalTx(*tx, nHeight, nLockTimeCutoff)) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-txns-nonfinal", "non-final transaction"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal", "non-final transaction"); } } @@ -3480,7 +3492,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c CScript expect = CScript() << nHeight; if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || !std::equal(expect.begin(), expect.end(), block.vtx[0]->vin[0].scriptSig.begin())) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-cb-height", "block height mismatch in coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-height", "block height mismatch in coinbase"); } } @@ -3502,11 +3514,11 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // already does not permit it, it is impossible to trigger in the // witness tree. if (block.vtx[0]->vin[0].scriptWitness.stack.size() != 1 || block.vtx[0]->vin[0].scriptWitness.stack[0].size() != 32) { - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-witness-nonce-size", strprintf("%s : invalid witness reserved value size", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-witness-nonce-size", strprintf("%s : invalid witness reserved value size", __func__)); } CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0]->vin[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); if (memcmp(hashWitness.begin(), &block.vtx[0]->vout[commitpos].scriptPubKey[6], 32)) { - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "bad-witness-merkle-match", strprintf("%s : witness merkle commitment mismatch", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-witness-merkle-match", strprintf("%s : witness merkle commitment mismatch", __func__)); } fHaveWitness = true; } @@ -3516,7 +3528,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c if (!fHaveWitness) { for (const auto& tx : block.vtx) { if (tx->HasWitness()) { - return state.Invalid(ValidationInvalidReason::BLOCK_MUTATED, false, REJECT_INVALID, "unexpected-witness", strprintf("%s : unexpected witness data found", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "unexpected-witness", strprintf("%s : unexpected witness data found", __func__)); } } } @@ -3528,13 +3540,13 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // the block hash, so we couldn't mark the block as permanently // failed). if (GetBlockWeight(block) > MAX_BLOCK_WEIGHT) { - return state.Invalid(ValidationInvalidReason::CONSENSUS, false, REJECT_INVALID, "bad-blk-weight", strprintf("%s : weight limit failed", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-weight", strprintf("%s : weight limit failed", __func__)); } return true; } -bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate @@ -3547,8 +3559,10 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState pindex = miSelf->second; if (ppindex) *ppindex = pindex; - if (pindex->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(ValidationInvalidReason::CACHED_INVALID, error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); + if (pindex->nStatus & BLOCK_FAILED_MASK) { + LogPrintf("ERROR: %s: block %s is marked invalid\n", __func__, hash.ToString()); + return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate"); + } return true; } @@ -3558,11 +3572,15 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState // Get prev block index CBlockIndex* pindexPrev = nullptr; BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); - if (mi == m_block_index.end()) - return state.Invalid(ValidationInvalidReason::BLOCK_MISSING_PREV, error("%s: prev block not found", __func__), 0, "prev-blk-not-found"); + if (mi == m_block_index.end()) { + LogPrintf("ERROR: %s: prev block not found\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); + } pindexPrev = (*mi).second; - if (pindexPrev->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_PREV, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { + LogPrintf("ERROR: %s: prev block invalid\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); + } if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); @@ -3599,7 +3617,8 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState setDirtyBlockIndex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } - return state.Invalid(ValidationInvalidReason::BLOCK_INVALID_PREV, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + LogPrintf("ERROR: %s: prev block invalid\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } } } @@ -3614,9 +3633,8 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, CValidationState } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex, CBlockHeader *first_invalid) +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { - if (first_invalid != nullptr) first_invalid->SetNull(); { LOCK(cs_main); for (const CBlockHeader& header : headers) { @@ -3625,7 +3643,6 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { - if (first_invalid) *first_invalid = header; return false; } if (ppindex) { @@ -3661,7 +3678,7 @@ static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChai } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3711,8 +3728,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali if (!CheckBlock(block, state, chainparams.GetConsensus()) || !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { - assert(IsBlockReason(state.GetReason())); - if (state.IsInvalid() && state.GetReason() != ValidationInvalidReason::BLOCK_MUTATED) { + if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); } @@ -3751,7 +3767,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons { CBlockIndex *pindex = nullptr; if (fNewBlock) *fNewBlock = false; - CValidationState state; + BlockValidationState state; // CheckBlock() does not support multi-threaded block validation because CBlock::fChecked can cause data race. // Therefore, the following critical section must include the CheckBlock() call as well. @@ -3772,14 +3788,14 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons NotifyHeaderTip(); - CValidationState state; // Only used to report errors, not invalidity - ignore it + BlockValidationState state; // Only used to report errors, not invalidity - ignore it if (!::ChainstateActive().ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed (%s)", __func__, FormatStateMessage(state)); return true; } -bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) { AssertLockHeld(cs_main); assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); @@ -3890,7 +3906,7 @@ static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPr /* This function is called from the RPC code for pruneblockchain */ void PruneBlockFilesManual(int nManualPruneHeight) { - CValidationState state; + BlockValidationState state; const CChainParams& chainparams = Params(); if (!::ChainstateActive().FlushStateToDisk( chainparams, state, FlushStateMode::NONE, nManualPruneHeight)) { @@ -4186,7 +4202,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; - CValidationState state; + BlockValidationState state; int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ for (pindex = ::ChainActive().Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { @@ -4430,7 +4446,7 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) } // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 - CValidationState state; + BlockValidationState state; // Loop until the tip is below nHeight, or we reach a pruned block. while (!ShutdownRequested()) { { @@ -4498,7 +4514,7 @@ bool RewindBlockIndex(const CChainParams& params) { // FlushStateToDisk can possibly read ::ChainActive(). Be conservative // and skip it here, we're about to -reindex-chainstate anyway, so // it'll get called a bunch real soon. - CValidationState state; + BlockValidationState state; if (!::ChainstateActive().FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", FormatStateMessage(state)); return false; @@ -4650,7 +4666,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // process in case the block isn't known yet CBlockIndex* pindex = LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { - CValidationState state; + BlockValidationState state; if (::ChainstateActive().AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { nLoaded++; } @@ -4664,7 +4680,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { - CValidationState state; + BlockValidationState state; if (!ActivateBestChain(state, chainparams)) { break; } @@ -4687,7 +4703,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); - CValidationState dummy; + BlockValidationState dummy; if (::ChainstateActive().AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; @@ -4964,10 +4980,10 @@ bool LoadMempool(CTxMemPool& pool) if (amountdelta) { pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } - CValidationState state; + TxValidationState state; if (nTime + nExpiryTimeout > nNow) { LOCK(cs_main); - AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nullptr /* pfMissingInputs */, nTime, + AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nTime, nullptr /* plTxnReplaced */, false /* bypass_limits */, 0 /* nAbsurdFee */, false /* test_accept */); if (state.IsValid()) { @@ -5073,7 +5089,7 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate; } - return pindex->nChainTx / fTxTotal; + return std::min<double>(pindex->nChainTx / fTxTotal, 1.0); } class CMainCleanup diff --git a/src/validation.h b/src/validation.h index 9140bf16e1..31721cfbf7 100644 --- a/src/validation.h +++ b/src/validation.h @@ -21,6 +21,7 @@ #include <txmempool.h> // For CTxMemPool::cs #include <txdb.h> #include <versionbits.h> +#include <serialize.h> #include <algorithm> #include <atomic> @@ -32,6 +33,7 @@ #include <vector> class CChainState; +class BlockValidationState; class CBlockIndex; class CBlockTreeDB; class CBlockUndo; @@ -41,7 +43,7 @@ class CConnman; class CScriptCheck; class CBlockPolicyEstimator; class CTxMemPool; -class CValidationState; +class TxValidationState; struct ChainTxData; struct DisconnectedBlockTransactions; @@ -75,8 +77,8 @@ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB -/** Maximum number of script-checking threads allowed */ -static const int MAX_SCRIPTCHECK_THREADS = 16; +/** Maximum number of dedicated script-checking threads allowed */ +static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; /** Number of blocks that can be requested at any given time from a single peer. */ @@ -145,7 +147,10 @@ extern std::condition_variable g_best_block_cv; extern uint256 g_best_block; extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; -extern int nScriptCheckThreads; +/** Whether there are dedicated script-checking threads running. + * False indicates all script checking is done on the main threadMessageHandler thread. + */ +extern bool g_parallel_script_checks; extern bool fRequireStandard; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; @@ -221,9 +226,8 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons * @param[out] state This may be set to an Error state if any error occurred processing them * @param[in] chainparams The params for the chain we want to connect to * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers - * @param[out] first_invalid First header that fails validation, if one exists */ -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); /** Open a block file (blk?????.dat) */ FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); @@ -248,7 +252,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::P * May not be called with cs_main held. May not be called in a * validationinterface callback. */ -bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); +bool ActivateBestChain(BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ @@ -272,8 +276,8 @@ void PruneBlockFilesManual(int nManualPruneHeight); /** (try to) add transaction to memory pool * plTxnReplaced will be appended to with all transactions replaced from mempool **/ -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, - bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced, +bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount nAbsurdFee, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get the BIP9 state for a given deployment at the current tip. */ @@ -368,10 +372,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); /** Functions for validating blocks and updating the block tree */ /** Context-independent validity checks */ -bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); +bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Check a block is completely valid from start to finish (only works on top of our current best block) */ -bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Check whether witness commitments are required for a block, and whether to enforce NULLDUMMY (BIP 147) rules. * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ @@ -488,7 +492,7 @@ public: */ bool AcceptBlockHeader( const CBlockHeader& block, - CValidationState& state, + BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; @@ -652,7 +656,7 @@ public: */ bool FlushStateToDisk( const CChainParams& chainparams, - CValidationState &state, + BlockValidationState &state, FlushStateMode mode, int nManualPruneHeight = 0); @@ -678,23 +682,23 @@ public: * @returns true unless a system error occurred */ bool ActivateBestChain( - CValidationState& state, + BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) LOCKS_EXCLUDED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); - bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + bool ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. - bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); // Manual block validity manipulation: - bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); - bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Replay blocks that aren't fully applied to the database. */ @@ -720,10 +724,10 @@ public: bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); private: - bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); - bool ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); - void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -738,10 +742,10 @@ private: * May not be called in a * validationinterface callback. */ -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); +bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); /** Mark a block as invalid. */ -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); +bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -777,14 +781,6 @@ extern VersionBitsCache versionbitscache; */ int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** Reject codes greater or equal to this can be returned by AcceptToMemPool - * for transactions, to signal internal conditions. They cannot and should not - * be sent over the P2P network. - */ -static const unsigned int REJECT_INTERNAL = 0x100; -/** Too high fee. Can not be triggered by P2P transactions */ -static const unsigned int REJECT_HIGHFEE = 0x100; - /** Get block file info entry for one block file */ CBlockFileInfo* GetBlockFileInfo(size_t n); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index cf4a529a6d..663308bae9 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -29,10 +29,10 @@ struct MainSignalsInstance { boost::signals2::signal<void (const CBlockIndex *, const CBlockIndex *, bool fInitialDownload)> UpdatedBlockTip; boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; - boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const std::shared_ptr<const CBlock>&, const CBlockIndex* pindex)> BlockDisconnected; boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> ChainStateFlushed; - boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; + boost::signals2::signal<void (const CBlock&, const BlockValidationState&)> BlockChecked; boost::signals2::signal<void (const CBlockIndex *, const std::shared_ptr<const CBlock>&)> NewPoWValidBlock; // We are not allowed to assume the scheduler only runs in one thread, @@ -92,7 +92,7 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { conns.UpdatedBlockTip = g_signals.m_internals->UpdatedBlockTip.connect(std::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); conns.TransactionAddedToMempool = g_signals.m_internals->TransactionAddedToMempool.connect(std::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, std::placeholders::_1)); conns.BlockConnected = g_signals.m_internals->BlockConnected.connect(std::bind(&CValidationInterface::BlockConnected, pwalletIn, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); - conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1)); + conns.BlockDisconnected = g_signals.m_internals->BlockDisconnected.connect(std::bind(&CValidationInterface::BlockDisconnected, pwalletIn, std::placeholders::_1, std::placeholders::_2)); conns.TransactionRemovedFromMempool = g_signals.m_internals->TransactionRemovedFromMempool.connect(std::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, std::placeholders::_1)); conns.ChainStateFlushed = g_signals.m_internals->ChainStateFlushed.connect(std::bind(&CValidationInterface::ChainStateFlushed, pwalletIn, std::placeholders::_1)); conns.BlockChecked = g_signals.m_internals->BlockChecked.connect(std::bind(&CValidationInterface::BlockChecked, pwalletIn, std::placeholders::_1, std::placeholders::_2)); @@ -156,9 +156,10 @@ void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, c }); } -void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { - m_internals->m_schedulerClient.AddToProcessQueue([pblock, this] { - m_internals->BlockDisconnected(pblock); +void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +{ + m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, this] { + m_internals->BlockDisconnected(pblock, pindex); }); } @@ -168,7 +169,7 @@ void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { }); } -void CMainSignals::BlockChecked(const CBlock& block, const CValidationState& state) { +void CMainSignals::BlockChecked(const CBlock& block, const BlockValidationState& state) { m_internals->BlockChecked(block, state); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 3ce617b827..6a8059a6a0 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -13,12 +13,12 @@ #include <memory> extern CCriticalSection cs_main; +class BlockValidationState; class CBlock; class CBlockIndex; struct CBlockLocator; class CConnman; class CValidationInterface; -class CValidationState; class uint256; class CScheduler; class CTxMemPool; @@ -114,7 +114,7 @@ protected: * * Called on a background thread. */ - virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} + virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) {} /** * Notifies listeners of the new active block chain on-disk. * @@ -134,11 +134,11 @@ protected: virtual void ChainStateFlushed(const CBlockLocator &locator) {} /** * Notifies listeners of a block validation result. - * If the provided CValidationState IsValid, the provided block + * If the provided BlockValidationState IsValid, the provided block * is guaranteed to be the current best block at the time the * callback was generated (not necessarily now) */ - virtual void BlockChecked(const CBlock&, const CValidationState&) {} + virtual void BlockChecked(const CBlock&, const BlockValidationState&) {} /** * Notifies listeners that a block which builds directly on our current tip * has been received and connected to the headers tree, though not validated yet */ @@ -178,9 +178,9 @@ public: void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &); void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); - void BlockDisconnected(const std::shared_ptr<const CBlock> &); + void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); void ChainStateFlushed(const CBlockLocator &); - void BlockChecked(const CBlock&, const CValidationState&); + void BlockChecked(const CBlock&, const BlockValidationState&); void NewPoWValidBlock(const CBlockIndex *, const std::shared_ptr<const CBlock>&); }; diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 92a290530c..fca4b75c45 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,13 +5,12 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include <optional.h> #include <policy/feerate.h> #include <policy/fees.h> #include <primitives/transaction.h> #include <wallet/wallet.h> -#include <boost/optional.hpp> - const int DEFAULT_MIN_DEPTH = 0; const int DEFAULT_MAX_DEPTH = 9999999; @@ -22,7 +21,7 @@ public: //! Custom change destination, if not set an address is generated CTxDestination destChange; //! Override the default change type if set, ignored if destChange is set - boost::optional<OutputType> m_change_type; + Optional<OutputType> m_change_type; //! If false, allows unselected inputs, but requires all selected inputs be used bool fAllowOtherInputs; //! Includes watch only addresses which are solvable @@ -30,11 +29,11 @@ public: //! Override automatic min/max checks on fee, m_feerate must be set if true bool fOverrideFeeRate; //! Override the wallet's m_pay_tx_fee if set - boost::optional<CFeeRate> m_feerate; + Optional<CFeeRate> m_feerate; //! Override the default confirmation target if set - boost::optional<unsigned int> m_confirm_target; + Optional<unsigned int> m_confirm_target; //! Override the wallet's m_signal_rbf if set - boost::optional<bool> m_signal_bip125_rbf; + Optional<bool> m_signal_bip125_rbf; //! Avoid partial use of funds sent to a given address bool m_avoid_partial_spends; //! Forbids inclusion of dirty (previously used) addresses diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 8a37f374a1..870e235964 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,11 +4,10 @@ #include <wallet/coinselection.h> +#include <optional.h> #include <util/system.h> #include <util/moneystr.h> -#include <boost/optional.hpp> - // Descending order comparator struct { bool operator()(const OutputGroup& a, const OutputGroup& b) const @@ -219,7 +218,7 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group nValueRet = 0; // List of values less than target - boost::optional<OutputGroup> lowest_larger; + Optional<OutputGroup> lowest_larger; std::vector<OutputGroup> applicable_groups; CAmount nTotalLower = 0; diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index f6179aa298..b50f00e7d1 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -23,7 +23,7 @@ int CCrypter::BytesToKeySHA512AES(const std::vector<unsigned char>& chSalt, cons unsigned char buf[CSHA512::OUTPUT_SIZE]; CSHA512 di; - di.Write((const unsigned char*)strKeyData.c_str(), strKeyData.size()); + di.Write((const unsigned char*)strKeyData.data(), strKeyData.size()); di.Write(chSalt.data(), chSalt.size()); di.Finalize(buf); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index e48eee6c2c..79825847c8 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -166,10 +166,9 @@ BerkeleyEnvironment::~BerkeleyEnvironment() bool BerkeleyEnvironment::Open(bool retry) { - if (fDbEnvInit) + if (fDbEnvInit) { return true; - - boost::this_thread::interruption_point(); + } fs::path pathIn = strPath; TryCreateDirectories(pathIn); @@ -238,14 +237,12 @@ bool BerkeleyEnvironment::Open(bool retry) return true; } -//! Construct an in-memory mock Berkeley environment for testing and as a place-holder for g_dbenvs emplace +//! Construct an in-memory mock Berkeley environment for testing BerkeleyEnvironment::BerkeleyEnvironment() { Reset(); - boost::this_thread::interruption_point(); - - LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::MakeMock\n"); dbenv->set_cachesize(1, 0, 1); dbenv->set_lg_bsize(10485760 * 4); @@ -263,8 +260,9 @@ BerkeleyEnvironment::BerkeleyEnvironment() DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); - if (ret > 0) + if (ret > 0) { throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret)); + } fDbEnvInit = true; fMockDb = true; @@ -767,7 +765,7 @@ void BerkeleyEnvironment::Flush(bool fShutdown) { int64_t nStart = GetTimeMillis(); // Flush log data to the actual data file on all files that are not in use - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started"); if (!fDbEnvInit) return; { @@ -776,21 +774,21 @@ void BerkeleyEnvironment::Flush(bool fShutdown) while (mi != mapFileUseCount.end()) { std::string strFile = (*mi).first; int nRefCount = (*mi).second; - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile); dbenv->txn_checkpoint(0, 0, 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s detach\n", strFile); if (!fMockDb) dbenv->lsn_reset(strFile.c_str(), 0); - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: %s closed\n", strFile); mapFileUseCount.erase(mi++); } else mi++; } - LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); + LogPrint(BCLog::WALLETDB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { @@ -830,7 +828,7 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); if (mi != env->mapFileUseCount.end()) { - LogPrint(BCLog::DB, "Flushing %s\n", strFile); + LogPrint(BCLog::WALLETDB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained @@ -838,7 +836,7 @@ bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database) env->CheckpointLSN(strFile); env->mapFileUseCount.erase(mi++); - LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + LogPrint(BCLog::WALLETDB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } } diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index fbcf8d4de7..8f0b495ac4 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <consensus/validation.h> #include <interfaces/chain.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> @@ -17,7 +16,7 @@ //! Check whether transaction has descendant in wallet or mempool, or has been //! mined, or conflicts with a mined transaction. Return a feebumper::Result. -static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWalletTx& wtx, std::vector<std::string>& errors) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { if (wallet.HasWalletSpend(wtx.GetHash())) { errors.push_back("Transaction has descendants in the wallet"); @@ -31,7 +30,7 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai } } - if (wtx.GetDepthInMainChain(locked_chain) != 0) { + if (wtx.GetDepthInMainChain() != 0) { errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); return feebumper::Result::WALLET_ERROR; } @@ -147,7 +146,7 @@ bool TransactionCanBeBumped(const CWallet& wallet, const uint256& txid) if (wtx == nullptr) return false; std::vector<std::string> errors_dummy; - feebumper::Result res = PreconditionChecks(*locked_chain, wallet, *wtx, errors_dummy); + feebumper::Result res = PreconditionChecks(wallet, *wtx, errors_dummy); return res == feebumper::Result::OK; } @@ -166,7 +165,7 @@ Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, co } const CWalletTx& wtx = it->second; - Result result = PreconditionChecks(*locked_chain, *wallet, wtx, errors); + Result result = PreconditionChecks(*wallet, wtx, errors); if (result != Result::OK) { return result; } @@ -292,7 +291,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } const CWalletTx& wtx = it->second; - Result result = PreconditionChecks(*locked_chain, wallet, wtx, errors); + Result result = PreconditionChecks(wallet, wtx, errors); if (result != Result::OK) { return result; } @@ -383,7 +382,7 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti CWalletTx& oldWtx = it->second; // make sure the transaction still has no descendants and hasn't been mined in the meantime - Result result = PreconditionChecks(*locked_chain, wallet, oldWtx, errors); + Result result = PreconditionChecks(wallet, oldWtx, errors); if (result != Result::OK) { return result; } @@ -393,21 +392,10 @@ Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransacti mapValue_t mapValue = oldWtx.mapValue; mapValue["replaces_txid"] = oldWtx.GetHash().ToString(); - CValidationState state; - if (!wallet.CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, state)) { - // NOTE: CommitTransaction never returns false, so this should never happen. - errors.push_back(strprintf("The transaction was rejected: %s", FormatStateMessage(state))); - return Result::WALLET_ERROR; - } - - bumped_txid = tx->GetHash(); - if (state.IsInvalid()) { - // This can happen if the mempool rejected the transaction. Report - // what happened in the "errors" response. - errors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); - } + wallet.CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm); // mark the original tx as bumped + bumped_txid = tx->GetHash(); if (!wallet.MarkReplaced(oldWtx.GetHash(), bumped_txid)) { // TODO: see if JSON-RPC has a standard way of returning a response // along with an exception. It would be good to return information about diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 3657a157b6..a8b3df1f2e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -6,6 +6,7 @@ #include <init.h> #include <interfaces/chain.h> #include <net.h> +#include <node/context.h> #include <outputtype.h> #include <util/moneystr.h> #include <util/system.h> @@ -25,8 +26,8 @@ public: //! Wallets parameter interaction bool ParameterInteraction() const override; - //! Add wallets that should be opened to list of init interfaces. - void Construct(InitInterfaces& interfaces) const override; + //! Add wallets that should be opened to list of chain clients. + void Construct(NodeContext& node) const override; }; const WalletInitInterface& g_wallet_init_interface = WalletInit(); @@ -125,12 +126,12 @@ bool WalletInit::ParameterInteraction() const return true; } -void WalletInit::Construct(InitInterfaces& interfaces) const +void WalletInit::Construct(NodeContext& node) const { if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { LogPrintf("Wallet disabled!\n"); return; } gArgs.SoftSetArg("-wallet", ""); - interfaces.chain_clients.emplace_back(interfaces::MakeWalletClient(*interfaces.chain, gArgs.GetArgs("-wallet"))); + node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, gArgs.GetArgs("-wallet"))); } diff --git a/src/wallet/ismine.cpp b/src/wallet/ismine.cpp deleted file mode 100644 index 029b922785..0000000000 --- a/src/wallet/ismine.cpp +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <wallet/ismine.h> - -#include <key.h> -#include <script/script.h> -#include <script/signingprovider.h> -#include <wallet/wallet.h> - -typedef std::vector<unsigned char> valtype; - -namespace { - -/** - * This is an enum that tracks the execution context of a script, similar to - * SigVersion in script/interpreter. It is separate however because we want to - * distinguish between top-level scriptPubKey execution and P2SH redeemScript - * execution (a distinction that has no impact on consensus rules). - */ -enum class IsMineSigVersion -{ - TOP = 0, //!< scriptPubKey execution - P2SH = 1, //!< P2SH redeemScript - WITNESS_V0 = 2, //!< P2WSH witness script execution -}; - -/** - * This is an internal representation of isminetype + invalidity. - * Its order is significant, as we return the max of all explored - * possibilities. - */ -enum class IsMineResult -{ - NO = 0, //!< Not ours - WATCH_ONLY = 1, //!< Included in watch-only balance - SPENDABLE = 2, //!< Included in all balances - INVALID = 3, //!< Not spendable by anyone (uncompressed pubkey in segwit, P2SH inside P2SH or witness, witness inside witness) -}; - -bool PermitsUncompressed(IsMineSigVersion sigversion) -{ - return sigversion == IsMineSigVersion::TOP || sigversion == IsMineSigVersion::P2SH; -} - -bool HaveKeys(const std::vector<valtype>& pubkeys, const CWallet& keystore) -{ - for (const valtype& pubkey : pubkeys) { - CKeyID keyID = CPubKey(pubkey).GetID(); - if (!keystore.HaveKey(keyID)) return false; - } - return true; -} - -IsMineResult IsMineInner(const CWallet& keystore, const CScript& scriptPubKey, IsMineSigVersion sigversion) -{ - IsMineResult ret = IsMineResult::NO; - - std::vector<valtype> vSolutions; - txnouttype whichType = Solver(scriptPubKey, vSolutions); - - CKeyID keyID; - switch (whichType) - { - case TX_NONSTANDARD: - case TX_NULL_DATA: - case TX_WITNESS_UNKNOWN: - break; - case TX_PUBKEY: - keyID = CPubKey(vSolutions[0]).GetID(); - if (!PermitsUncompressed(sigversion) && vSolutions[0].size() != 33) { - return IsMineResult::INVALID; - } - if (keystore.HaveKey(keyID)) { - ret = std::max(ret, IsMineResult::SPENDABLE); - } - break; - case TX_WITNESS_V0_KEYHASH: - { - if (sigversion == IsMineSigVersion::WITNESS_V0) { - // P2WPKH inside P2WSH is invalid. - return IsMineResult::INVALID; - } - if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { - // We do not support bare witness outputs unless the P2SH version of it would be - // acceptable as well. This protects against matching before segwit activates. - // This also applies to the P2WSH case. - break; - } - ret = std::max(ret, IsMineInner(keystore, GetScriptForDestination(PKHash(uint160(vSolutions[0]))), IsMineSigVersion::WITNESS_V0)); - break; - } - case TX_PUBKEYHASH: - keyID = CKeyID(uint160(vSolutions[0])); - if (!PermitsUncompressed(sigversion)) { - CPubKey pubkey; - if (keystore.GetPubKey(keyID, pubkey) && !pubkey.IsCompressed()) { - return IsMineResult::INVALID; - } - } - if (keystore.HaveKey(keyID)) { - ret = std::max(ret, IsMineResult::SPENDABLE); - } - break; - case TX_SCRIPTHASH: - { - if (sigversion != IsMineSigVersion::TOP) { - // P2SH inside P2WSH or P2SH is invalid. - return IsMineResult::INVALID; - } - CScriptID scriptID = CScriptID(uint160(vSolutions[0])); - CScript subscript; - if (keystore.GetCScript(scriptID, subscript)) { - ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::P2SH)); - } - break; - } - case TX_WITNESS_V0_SCRIPTHASH: - { - if (sigversion == IsMineSigVersion::WITNESS_V0) { - // P2WSH inside P2WSH is invalid. - return IsMineResult::INVALID; - } - if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { - break; - } - uint160 hash; - CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(hash.begin()); - CScriptID scriptID = CScriptID(hash); - CScript subscript; - if (keystore.GetCScript(scriptID, subscript)) { - ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::WITNESS_V0)); - } - break; - } - - case TX_MULTISIG: - { - // Never treat bare multisig outputs as ours (they can still be made watchonly-though) - if (sigversion == IsMineSigVersion::TOP) { - break; - } - - // Only consider transactions "mine" if we own ALL the - // keys involved. Multi-signature transactions that are - // partially owned (somebody else has a key that can spend - // them) enable spend-out-from-under-you attacks, especially - // in shared-wallet situations. - std::vector<valtype> keys(vSolutions.begin()+1, vSolutions.begin()+vSolutions.size()-1); - if (!PermitsUncompressed(sigversion)) { - for (size_t i = 0; i < keys.size(); i++) { - if (keys[i].size() != 33) { - return IsMineResult::INVALID; - } - } - } - if (HaveKeys(keys, keystore)) { - ret = std::max(ret, IsMineResult::SPENDABLE); - } - break; - } - } - - if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { - ret = std::max(ret, IsMineResult::WATCH_ONLY); - } - return ret; -} - -} // namespace - -isminetype IsMine(const CWallet& keystore, const CScript& scriptPubKey) -{ - switch (IsMineInner(keystore, scriptPubKey, IsMineSigVersion::TOP)) { - case IsMineResult::INVALID: - case IsMineResult::NO: - return ISMINE_NO; - case IsMineResult::WATCH_ONLY: - return ISMINE_WATCH_ONLY; - case IsMineResult::SPENDABLE: - return ISMINE_SPENDABLE; - } - assert(false); -} - -isminetype IsMine(const CWallet& keystore, const CTxDestination& dest) -{ - CScript script = GetScriptForDestination(dest); - return IsMine(keystore, script); -} diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 41555fcb93..0bc6c90354 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -28,9 +28,6 @@ enum isminetype : unsigned int /** used for bitflags of isminetype */ typedef uint8_t isminefilter; -isminetype IsMine(const CWallet& wallet, const CScript& scriptPubKey); -isminetype IsMine(const CWallet& wallet, const CTxDestination& dest); - /** * Cachable amount subdivided into watchonly and spendable parts. */ diff --git a/src/wallet/psbtwallet.cpp b/src/wallet/psbtwallet.cpp index 721a244afb..aa13cacca4 100644 --- a/src/wallet/psbtwallet.cpp +++ b/src/wallet/psbtwallet.cpp @@ -39,12 +39,12 @@ TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& ps return TransactionError::SIGHASH_MISMATCH; } - complete &= SignPSBTInput(HidingSigningProvider(pwallet, !sign, !bip32derivs), psbtx, i, sighash_type); + complete &= SignPSBTInput(HidingSigningProvider(pwallet->GetSigningProvider(), !sign, !bip32derivs), psbtx, i, sighash_type); } // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { - UpdatePSBTOutput(HidingSigningProvider(pwallet, true, !bip32derivs), psbtx, i); + UpdatePSBTOutput(HidingSigningProvider(pwallet->GetSigningProvider(), true, !bip32derivs), psbtx, i); } return TransactionError::OK; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 1cd4cb93b4..d95481500d 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -23,23 +23,10 @@ #include <tuple> #include <boost/algorithm/string.hpp> -#include <boost/date_time/posix_time/posix_time.hpp> #include <univalue.h> -int64_t static DecodeDumpTime(const std::string &str) { - static const boost::posix_time::ptime epoch = boost::posix_time::from_time_t(0); - static const std::locale loc(std::locale::classic(), - new boost::posix_time::time_input_facet("%Y-%m-%dT%H:%M:%SZ")); - std::istringstream iss(str); - iss.imbue(loc); - boost::posix_time::ptime ptime(boost::date_time::not_a_date_time); - iss >> ptime; - if (ptime.is_not_a_date_time()) - return 0; - return (ptime - epoch).total_seconds(); -} std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; @@ -67,11 +54,11 @@ static std::string DecodeDumpString(const std::string &str) { return ret.str(); } -static bool GetWalletAddressesForKey(CWallet* const pwallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static bool GetWalletAddressesForKey(LegacyScriptPubKeyMan* spk_man, CWallet* const pwallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { bool fLabelFound = false; CKey key; - pwallet->GetKey(keyid, key); + spk_man->GetKey(keyid, key); for (const auto& dest : GetAllDestinationsForKey(key.GetPubKey())) { if (pwallet->mapAddressBook.count(dest)) { if (!strAddr.empty()) { @@ -138,6 +125,8 @@ UniValue importprivkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } + EnsureLegacyScriptPubKeyMan(*wallet); + WalletRescanReserver reserver(pwallet); bool fRescan = true; { @@ -170,7 +159,7 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!key.IsValid()) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); + CHECK_NONFATAL(key.VerifyPubKey(pubkey)); CKeyID vchAddress = pubkey.GetID(); { pwallet->MarkDirty(); @@ -264,6 +253,7 @@ UniValue importaddress(const JSONRPCRequest& request) }, }.Check(request); + EnsureLegacyScriptPubKeyMan(*pwallet); std::string strLabel; if (!request.params[1].isNull()) @@ -326,7 +316,7 @@ UniValue importaddress(const JSONRPCRequest& request) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } } @@ -364,28 +354,26 @@ UniValue importprunedfunds(const JSONRPCRequest& request) //Search partial merkle tree in proof for our transaction and index in valid block std::vector<uint256> vMatch; std::vector<unsigned int> vIndex; - unsigned int txnIndex = 0; - if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) == merkleBlock.header.hashMerkleRoot) { - - auto locked_chain = pwallet->chain().lock(); - if (locked_chain->getBlockHeight(merkleBlock.header.GetHash()) == nullopt) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); - } - - std::vector<uint256>::const_iterator it; - if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx))==vMatch.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); - } + if (merkleBlock.txn.ExtractMatches(vMatch, vIndex) != merkleBlock.header.hashMerkleRoot) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); + } - txnIndex = vIndex[it - vMatch.begin()]; + auto locked_chain = pwallet->chain().lock(); + Optional<int> height = locked_chain->getBlockHeight(merkleBlock.header.GetHash()); + if (height == nullopt) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } - else { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Something wrong with merkleblock"); + + std::vector<uint256>::const_iterator it; + if ((it = std::find(vMatch.begin(), vMatch.end(), hashTx)) == vMatch.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction given doesn't exist in proof"); } - wtx.SetConf(CWalletTx::Status::CONFIRMED, merkleBlock.header.GetHash(), txnIndex); + unsigned int txnIndex = vIndex[it - vMatch.begin()]; + + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *height, merkleBlock.header.GetHash(), txnIndex); + wtx.m_confirm = confirm; - auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); if (pwallet->IsMine(*wtx.tx)) { @@ -466,6 +454,7 @@ UniValue importpubkey(const JSONRPCRequest& request) }, }.Check(request); + EnsureLegacyScriptPubKeyMan(*wallet); std::string strLabel; if (!request.params[1].isNull()) @@ -516,7 +505,7 @@ UniValue importpubkey(const JSONRPCRequest& request) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } } @@ -549,6 +538,8 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.Check(request); + EnsureLegacyScriptPubKeyMan(*wallet); + if (pwallet->chain().havePruned()) { // Exit early and print an error. // If a block is pruned after this check, we will import the key(s), @@ -598,7 +589,7 @@ UniValue importwallet(const JSONRPCRequest& request) continue; CKey key = DecodeSecret(vstr[0]); if (key.IsValid()) { - int64_t nTime = DecodeDumpTime(vstr[1]); + int64_t nTime = ParseISO8601DateTime(vstr[1]); std::string strLabel; bool fLabel = true; for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { @@ -617,7 +608,7 @@ UniValue importwallet(const JSONRPCRequest& request) } else if(IsHex(vstr[0])) { std::vector<unsigned char> vData(ParseHex(vstr[0])); CScript script = CScript(vData.begin(), vData.end()); - int64_t birth_time = DecodeDumpTime(vstr[1]); + int64_t birth_time = ParseISO8601DateTime(vstr[1]); scripts.push_back(std::pair<CScript, int64_t>(script, birth_time)); } } @@ -637,7 +628,7 @@ UniValue importwallet(const JSONRPCRequest& request) std::string label = std::get<3>(key_tuple); CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); + CHECK_NONFATAL(key.VerifyPubKey(pubkey)); CKeyID keyid = pubkey.GetID(); pwallet->WalletLogPrintf("Importing %s...\n", EncodeDestination(PKHash(keyid))); @@ -706,6 +697,8 @@ UniValue dumpprivkey(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); + auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -716,12 +709,12 @@ UniValue dumpprivkey(const JSONRPCRequest& request) if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } - auto keyid = GetKeyForDestination(*pwallet, dest); + auto keyid = GetKeyForDestination(spk_man, dest); if (keyid.IsNull()) { throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to a key"); } CKey vchSecret; - if (!pwallet->GetKey(keyid, vchSecret)) { + if (!spk_man.GetKey(keyid, vchSecret)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key for address " + strAddress + " is not known"); } return EncodeSecret(vchSecret); @@ -755,8 +748,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); + auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); + AssertLockHeld(spk_man.cs_wallet); EnsureWalletIsUnlocked(pwallet); @@ -778,10 +774,10 @@ UniValue dumpwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); std::map<CKeyID, int64_t> mapKeyBirth; - const std::map<CKeyID, int64_t>& mapKeyPool = pwallet->GetAllReserveKeys(); + const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); pwallet->GetKeyBirthTimes(*locked_chain, mapKeyBirth); - std::set<CScriptID> scripts = pwallet->GetCScripts(); + std::set<CScriptID> scripts = spk_man.GetCScripts(); // sort time/key pairs std::vector<std::pair<int64_t, CKeyID> > vKeyBirth; @@ -800,11 +796,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) file << "\n"; // add the base58check encoded extended master if the wallet uses HD - CKeyID seed_id = pwallet->GetHDChain().seed_id; + CKeyID seed_id = spk_man.GetHDChain().seed_id; if (!seed_id.IsNull()) { CKey seed; - if (pwallet->GetKey(seed_id, seed)) { + if (spk_man.GetKey(seed_id, seed)) { CExtKey masterKey; masterKey.SetSeed(seed.begin(), seed.size()); @@ -817,20 +813,20 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::string strAddr; std::string strLabel; CKey key; - if (pwallet->GetKey(keyid, key)) { + if (spk_man.GetKey(keyid, key)) { file << strprintf("%s %s ", EncodeSecret(key), strTime); - if (GetWalletAddressesForKey(pwallet, keyid, strAddr, strLabel)) { + if (GetWalletAddressesForKey(&spk_man, pwallet, keyid, strAddr, strLabel)) { file << strprintf("label=%s", strLabel); } else if (keyid == seed_id) { file << "hdseed=1"; } else if (mapKeyPool.count(keyid)) { file << "reserve=1"; - } else if (pwallet->mapKeyMetadata[keyid].hdKeypath == "s") { + } else if (spk_man.mapKeyMetadata[keyid].hdKeypath == "s") { file << "inactivehdseed=1"; } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(pwallet->mapKeyMetadata[keyid].key_origin.path) : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (spk_man.mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(spk_man.mapKeyMetadata[keyid].key_origin.path) : "")); } } file << "\n"; @@ -839,11 +835,11 @@ UniValue dumpwallet(const JSONRPCRequest& request) std::string create_time = "0"; std::string address = EncodeDestination(ScriptHash(scriptid)); // get birth times for scripts with metadata - auto it = pwallet->m_script_metadata.find(scriptid); - if (it != pwallet->m_script_metadata.end()) { + auto it = spk_man.m_script_metadata.find(scriptid); + if (it != spk_man.m_script_metadata.end()) { create_time = FormatISO8601DateTime(it->second.nCreateTime); } - if(pwallet->GetCScript(scriptid, script)) { + if(spk_man.GetCScript(scriptid, script)) { file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); file << strprintf(" # addr=%s\n", address); } @@ -899,7 +895,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TX_SCRIPTHASH: { if (script_ctx == ScriptContext::P2SH) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside another P2SH"); if (script_ctx == ScriptContext::WITNESS_V0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Trying to nest P2SH inside a P2WSH"); - assert(script_ctx == ScriptContext::TOP); + CHECK_NONFATAL(script_ctx == ScriptContext::TOP); CScriptID id = CScriptID(uint160(solverdata[0])); auto subscript = std::move(import_data.redeemscript); // Remove redeemscript from import_data to check for superfluous script later. if (!subscript) return "missing redeemscript"; @@ -1219,7 +1215,7 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con // Check whether we have any work to do for (const CScript& script : script_pub_keys) { - if (::IsMine(*pwallet, script) & ISMINE_SPENDABLE) { + if (pwallet->IsMine(script) & ISMINE_SPENDABLE) { throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script.begin(), script.end()) + "\")"); } } @@ -1339,6 +1335,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); + EnsureLegacyScriptPubKeyMan(*wallet); + const UniValue& requests = mainRequest.params[0]; //Default options @@ -1406,7 +1404,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) { auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - pwallet->ReacceptWalletTransactions(*locked_chain); + pwallet->ReacceptWalletTransactions(); } if (pwallet->IsAbortingRescan()) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e159866921..00742aed7a 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -4,11 +4,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <amount.h> -#include <consensus/validation.h> #include <core_io.h> -#include <init.h> #include <interfaces/chain.h> #include <key_io.h> +#include <node/context.h> #include <outputtype.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -69,7 +68,7 @@ static bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWall /** Checks if a CKey is in the given CWallet compressed or otherwise*/ -bool HaveKey(const CWallet& wallet, const CKey& key) +bool HaveKey(const SigningProvider& wallet, const CKey& key) { CKey key2; key2.Set(key.begin(), key.end(), !key.IsCompressed()); @@ -125,19 +124,29 @@ void EnsureWalletIsUnlocked(const CWallet* pwallet) } } +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet) +{ + LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); + } + return *spk_man; +} + static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, UniValue& entry) { - int confirms = wtx.GetDepthInMainChain(locked_chain); + int confirms = wtx.GetDepthInMainChain(); entry.pushKV("confirmations", confirms); if (wtx.IsCoinBase()) entry.pushKV("generated", true); if (confirms > 0) { entry.pushKV("blockhash", wtx.m_confirm.hashBlock.GetHex()); + entry.pushKV("blockheight", wtx.m_confirm.block_height); entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); - assert(found_block); + CHECK_NONFATAL(found_block); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); @@ -304,7 +313,7 @@ static UniValue setlabel(const JSONRPCRequest& request) std::string label = LabelFromValue(request.params[1]); - if (IsMine(*pwallet, dest)) { + if (pwallet->IsMine(dest)) { pwallet->SetAddressBook(dest, label, "receive"); } else { pwallet->SetAddressBook(dest, label, "send"); @@ -341,11 +350,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet strError = strprintf("Error: This transaction requires a transaction fee of at least %s", FormatMoney(nFeeRequired)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } - CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, state)) { - strError = strprintf("Error: The transaction was rejected! Reason given: %s", FormatStateMessage(state)); - throw JSONRPCError(RPC_WALLET_ERROR, strError); - } + pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); return tx; } @@ -555,9 +560,11 @@ static UniValue signmessage(const JSONRPCRequest& request) throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); } + const SigningProvider* provider = pwallet->GetSigningProvider(); + CKey key; CKeyID keyID(*pkhash); - if (!pwallet->GetKey(keyID, key)) { + if (!provider->GetKey(keyID, key)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key not available"); } @@ -615,7 +622,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } CScript scriptPubKey = GetScriptForDestination(dest); - if (!IsMine(*pwallet, scriptPubKey)) { + if (!pwallet->IsMine(scriptPubKey)) { throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); } @@ -634,7 +641,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) for (const CTxOut& txout : wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain(*locked_chain) >= nMinDepth) + if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } @@ -699,8 +706,8 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) for (const CTxOut& txout : wtx.tx->vout) { CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && IsMine(*pwallet, address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain(*locked_chain) >= nMinDepth) + if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { + if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } } @@ -926,12 +933,7 @@ static UniValue sendmany(const JSONRPCRequest& request) bool fCreated = pwallet->CreateTransaction(*locked_chain, vecSend, tx, nFeeRequired, nChangePosRet, strFailReason, coin_control); if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); - CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, state)) { - strFailReason = strprintf("Transaction commit failed:: %s", FormatStateMessage(state)); - throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); - } - + pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */); return tx->GetHash().GetHex(); } @@ -974,6 +976,8 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -990,7 +994,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) if (IsHex(keys_or_addrs[i].get_str()) && (keys_or_addrs[i].get_str().length() == 66 || keys_or_addrs[i].get_str().length() == 130)) { pubkeys.push_back(HexToPubKey(keys_or_addrs[i].get_str())); } else { - pubkeys.push_back(AddrToPubKey(pwallet, keys_or_addrs[i].get_str())); + pubkeys.push_back(AddrToPubKey(&spk_man, keys_or_addrs[i].get_str())); } } @@ -1003,7 +1007,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) // Construct using pay-to-script-hash: CScript inner; - CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, *pwallet, inner); + CTxDestination dest = AddAndGetMultisigDestination(required, pubkeys, output_type, spk_man, inner); pwallet->SetAddressBook(dest, label, "send"); UniValue result(UniValue::VOBJ); @@ -1060,7 +1064,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co continue; } - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < nMinDepth) continue; @@ -1074,7 +1078,7 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co continue; } - isminefilter mine = IsMine(*pwallet, address); + isminefilter mine = pwallet->IsMine(address); if(!(mine & filter)) continue; @@ -1298,7 +1302,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con for (const COutputEntry& s : listSent) { UniValue entry(UniValue::VOBJ); - if (involvesWatchonly || (::IsMine(*pwallet, s.destination) & ISMINE_WATCH_ONLY)) { + if (involvesWatchonly || (pwallet->IsMine(s.destination) & ISMINE_WATCH_ONLY)) { entry.pushKV("involvesWatchonly", true); } MaybePushAddress(entry, s.destination); @@ -1317,8 +1321,7 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con } // Received - if (listReceived.size() > 0 && wtx.GetDepthInMainChain(locked_chain) >= nMinDepth) - { + if (listReceived.size() > 0 && wtx.GetDepthInMainChain() >= nMinDepth) { for (const COutputEntry& r : listReceived) { std::string label; @@ -1329,15 +1332,15 @@ static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* con continue; } UniValue entry(UniValue::VOBJ); - if (involvesWatchonly || (::IsMine(*pwallet, r.destination) & ISMINE_WATCH_ONLY)) { + if (involvesWatchonly || (pwallet->IsMine(r.destination) & ISMINE_WATCH_ONLY)) { entry.pushKV("involvesWatchonly", true); } MaybePushAddress(entry, r.destination); if (wtx.IsCoinBase()) { - if (wtx.GetDepthInMainChain(locked_chain) < 1) + if (wtx.GetDepthInMainChain() < 1) entry.pushKV("category", "orphan"); - else if (wtx.IsImmatureCoinBase(locked_chain)) + else if (wtx.IsImmatureCoinBase()) entry.pushKV("category", "immature"); else entry.pushKV("category", "generate"); @@ -1365,6 +1368,7 @@ static const std::string TransactionDescriptionString() " \"generated\": xxx, (bool) Only present if transaction only input is a coinbase one.\n" " \"trusted\": xxx, (bool) Only present if we consider transaction to be trusted and so safe to spend from.\n" " \"blockhash\": \"hashvalue\", (string) The block hash containing the transaction.\n" + " \"blockheight\": n, (numeric) The block height containing the transaction.\n" " \"blockindex\": n, (numeric) The index of the transaction in the block that includes it.\n" " \"blocktime\": xxx, (numeric) The block time in seconds since epoch (1 Jan 1970 GMT).\n" " \"txid\": \"transactionid\", (string) The transaction id.\n" @@ -1601,7 +1605,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; - if (depth == -1 || tx.GetDepthInMainChain(*locked_chain) < depth) { + if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { ListTransactions(*locked_chain, pwallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1718,7 +1722,7 @@ static UniValue gettransaction(const JSONRPCRequest& request) } const CWalletTx& wtx = it->second; - CAmount nCredit = wtx.GetCredit(*locked_chain, filter); + CAmount nCredit = wtx.GetCredit(filter); CAmount nDebit = wtx.GetDebit(filter); CAmount nNet = nCredit - nDebit; CAmount nFee = (wtx.IsFromMe(filter) ? wtx.tx->GetValueOut() - nDebit : 0); @@ -1782,7 +1786,7 @@ static UniValue abandontransaction(const JSONRPCRequest& request) if (!pwallet->mapWallet.count(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid or non-wallet transaction id"); } - if (!pwallet->AbandonTransaction(*locked_chain, hash)) { + if (!pwallet->AbandonTransaction(hash)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not eligible for abandonment"); } @@ -2213,7 +2217,7 @@ static UniValue lockunspent(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout index out of bounds"); } - if (pwallet->IsSpent(*locked_chain, outpt.hash, outpt.n)) { + if (pwallet->IsSpent(outpt.hash, outpt.n)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected unspent output"); } @@ -2389,7 +2393,8 @@ static UniValue getbalances(const JSONRPCRequest& request) } balances.pushKV("mine", balances_mine); } - if (wallet.HaveWatchOnly()) { + auto spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (spk_man && spk_man->HaveWatchOnly()) { UniValue balances_watchonly{UniValue::VOBJ}; balances_watchonly.pushKV("trusted", ValueFromAmount(bal.m_watchonly_trusted)); balances_watchonly.pushKV("untrusted_pending", ValueFromAmount(bal.m_watchonly_untrusted_pending)); @@ -2459,7 +2464,15 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("txcount", (int)pwallet->mapWallet.size()); obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); obj.pushKV("keypoolsize", (int64_t)kpExternalSize); - CKeyID seed_id = pwallet->GetHDChain().seed_id; + + LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); + if (spk_man) { + CKeyID seed_id = spk_man->GetHDChain().seed_id; + if (!seed_id.IsNull()) { + obj.pushKV("hdseedid", seed_id.GetHex()); + } + } + if (pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { obj.pushKV("keypoolsize_hd_internal", (int64_t)(pwallet->GetKeyPoolSize() - kpExternalSize)); } @@ -2467,9 +2480,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("unlocked_until", pwallet->nRelockTime); } obj.pushKV("paytxfee", ValueFromAmount(pwallet->m_pay_tx_fee.GetFeePerK())); - if (!seed_id.IsNull()) { - obj.pushKV("hdseedid", seed_id.GetHex()); - } obj.pushKV("private_keys_enabled", !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); obj.pushKV("avoid_reuse", pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)); if (pwallet->IsScanning()) { @@ -2584,7 +2594,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) std::string error; std::vector<std::string> warning; - std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_interfaces->chain, location, error, warning); + std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_chain, location, error, warning); if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error); UniValue obj(UniValue::VOBJ); @@ -2710,7 +2720,7 @@ static UniValue createwallet(const JSONRPCRequest& request) std::string error; std::shared_ptr<CWallet> wallet; - WalletCreationStatus status = CreateWallet(*g_rpc_interfaces->chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); + WalletCreationStatus status = CreateWallet(*g_rpc_chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); switch (status) { case WalletCreationStatus::CREATION_FAILED: throw JSONRPCError(RPC_WALLET_ERROR, error); @@ -2930,22 +2940,23 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("label", i->second.name); } + const SigningProvider* provider = pwallet->GetSigningProvider(); if (scriptPubKey.IsPayToScriptHash()) { const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); CScript redeemScript; - if (pwallet->GetCScript(hash, redeemScript)) { + if (provider->GetCScript(hash, redeemScript)) { entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); // Now check if the redeemScript is actually a P2WSH script CTxDestination witness_destination; if (redeemScript.IsPayToWitnessScriptHash()) { bool extracted = ExtractDestination(redeemScript, witness_destination); - assert(extracted); + CHECK_NONFATAL(extracted); // Also return the witness script const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); CScriptID id; CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; - if (pwallet->GetCScript(id, witnessScript)) { + if (provider->GetCScript(id, witnessScript)) { entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); } } @@ -2955,7 +2966,7 @@ static UniValue listunspent(const JSONRPCRequest& request) CScriptID id; CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); CScript witnessScript; - if (pwallet->GetCScript(id, witnessScript)) { + if (provider->GetCScript(id, witnessScript)) { entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); } } @@ -2967,7 +2978,7 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("spendable", out.fSpendable); entry.pushKV("solvable", out.fSolvable); if (out.fSolvable) { - auto descriptor = InferDescriptor(scriptPubKey, *pwallet); + auto descriptor = InferDescriptor(scriptPubKey, *pwallet->GetLegacyScriptPubKeyMan()); entry.pushKV("desc", descriptor->ToString()); } if (avoid_reuse) entry.pushKV("reused", reused); @@ -3277,7 +3288,7 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) // Parse the prevtxs array ParsePrevouts(request.params[1], nullptr, coins); - return SignTransaction(mtx, pwallet, coins, request.params[2]); + return SignTransaction(mtx, &*pwallet->GetLegacyScriptPubKeyMan(), coins, request.params[2]); } static UniValue bumpfee(const JSONRPCRequest& request) @@ -3549,7 +3560,7 @@ UniValue rescanblockchain(const JSONRPCRequest& request) class DescribeWalletAddressVisitor : public boost::static_visitor<UniValue> { public: - CWallet * const pwallet; + const SigningProvider * const provider; void ProcessSubScript(const CScript& subscript, UniValue& obj) const { @@ -3585,7 +3596,7 @@ public: } } - explicit DescribeWalletAddressVisitor(CWallet* _pwallet) : pwallet(_pwallet) {} + explicit DescribeWalletAddressVisitor(const SigningProvider* _provider) : provider(_provider) {} UniValue operator()(const CNoDestination& dest) const { return UniValue(UniValue::VOBJ); } @@ -3594,7 +3605,7 @@ public: CKeyID keyID(pkhash); UniValue obj(UniValue::VOBJ); CPubKey vchPubKey; - if (pwallet && pwallet->GetPubKey(keyID, vchPubKey)) { + if (provider && provider->GetPubKey(keyID, vchPubKey)) { obj.pushKV("pubkey", HexStr(vchPubKey)); obj.pushKV("iscompressed", vchPubKey.IsCompressed()); } @@ -3606,7 +3617,7 @@ public: CScriptID scriptID(scripthash); UniValue obj(UniValue::VOBJ); CScript subscript; - if (pwallet && pwallet->GetCScript(scriptID, subscript)) { + if (provider && provider->GetCScript(scriptID, subscript)) { ProcessSubScript(subscript, obj); } return obj; @@ -3616,7 +3627,7 @@ public: { UniValue obj(UniValue::VOBJ); CPubKey pubkey; - if (pwallet && pwallet->GetPubKey(CKeyID(id), pubkey)) { + if (provider && provider->GetPubKey(CKeyID(id), pubkey)) { obj.pushKV("pubkey", HexStr(pubkey)); } return obj; @@ -3629,7 +3640,7 @@ public: CRIPEMD160 hasher; uint160 hash; hasher.Write(id.begin(), 32).Finalize(hash.begin()); - if (pwallet && pwallet->GetCScript(CScriptID(hash), subscript)) { + if (provider && provider->GetCScript(CScriptID(hash), subscript)) { ProcessSubScript(subscript, obj); } return obj; @@ -3642,8 +3653,12 @@ static UniValue DescribeWalletAddress(CWallet* pwallet, const CTxDestination& de { UniValue ret(UniValue::VOBJ); UniValue detail = DescribeAddress(dest); + const SigningProvider* provider = nullptr; + if (pwallet) { + provider = pwallet->GetSigningProvider(); + } ret.pushKVs(detail); - ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(pwallet), dest)); + ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider), dest)); return ret; } @@ -3732,13 +3747,14 @@ UniValue getaddressinfo(const JSONRPCRequest& request) CScript scriptPubKey = GetScriptForDestination(dest); ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); + const SigningProvider* provider = pwallet->GetSigningProvider(); - isminetype mine = IsMine(*pwallet, dest); + isminetype mine = pwallet->IsMine(dest); ret.pushKV("ismine", bool(mine & ISMINE_SPENDABLE)); - bool solvable = IsSolvable(*pwallet, scriptPubKey); + bool solvable = IsSolvable(*provider, scriptPubKey); ret.pushKV("solvable", solvable); if (solvable) { - ret.pushKV("desc", InferDescriptor(scriptPubKey, *pwallet)->ToString()); + ret.pushKV("desc", InferDescriptor(scriptPubKey, *provider)->ToString()); } ret.pushKV("iswatchonly", bool(mine & ISMINE_WATCH_ONLY)); UniValue detail = DescribeWalletAddress(pwallet, dest); @@ -3747,26 +3763,16 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ret.pushKV("label", pwallet->mapAddressBook[dest].name); } ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); - const CKeyMetadata* meta = nullptr; - CKeyID key_id = GetKeyForDestination(*pwallet, dest); - if (!key_id.IsNull()) { - auto it = pwallet->mapKeyMetadata.find(key_id); - if (it != pwallet->mapKeyMetadata.end()) { - meta = &it->second; - } - } - if (!meta) { - auto it = pwallet->m_script_metadata.find(CScriptID(scriptPubKey)); - if (it != pwallet->m_script_metadata.end()) { - meta = &it->second; - } - } - if (meta) { - ret.pushKV("timestamp", meta->nCreateTime); - if (meta->has_key_origin) { - ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); - ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); - ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + + ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(); + if (spk_man) { + if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { + ret.pushKV("timestamp", meta->nCreateTime); + if (meta->has_key_origin) { + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); + ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); + ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + } } } @@ -3824,7 +3830,7 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // address strings, but build a separate set as a precaution just in // case it does. bool unique = addresses.emplace(address).second; - assert(unique); + CHECK_NONFATAL(unique); // UniValue::pushKV checks if the key exists in O(N) // and since duplicate addresses are unexpected (checked with // std::set in O(log(N))), UniValue::__pushKV is used instead, @@ -3926,6 +3932,8 @@ UniValue sethdseed(const JSONRPCRequest& request) }, }.Check(request); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); } @@ -3951,22 +3959,22 @@ UniValue sethdseed(const JSONRPCRequest& request) CPubKey master_pub_key; if (request.params[1].isNull()) { - master_pub_key = pwallet->GenerateNewSeed(); + master_pub_key = spk_man.GenerateNewSeed(); } else { CKey key = DecodeSecret(request.params[1].get_str()); if (!key.IsValid()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key"); } - if (HaveKey(*pwallet, key)) { + if (HaveKey(spk_man, key)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Already have this key (either as an HD seed or as a loose private key)"); } - master_pub_key = pwallet->DeriveNewSeed(key); + master_pub_key = spk_man.DeriveNewSeed(key); } - pwallet->SetHDSeed(master_pub_key); - if (flush_key_pool) pwallet->NewKeyPool(); + spk_man.SetHDSeed(master_pub_key); + if (flush_key_pool) spk_man.NewKeyPool(); return NullUniValue; } @@ -4241,3 +4249,5 @@ void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) handlers.emplace_back(chain.handleRpc(commands[vcidx])); } + +interfaces::Chain* g_rpc_chain = nullptr; diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 1c0523c90b..becca455f6 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -12,6 +12,7 @@ class CRPCTable; class CWallet; class JSONRPCRequest; +class LegacyScriptPubKeyMan; class UniValue; struct PartiallySignedTransaction; class CTransaction; @@ -21,6 +22,12 @@ class Chain; class Handler; } +//! Pointer to chain interface that needs to be declared as a global to be +//! accessible loadwallet and createwallet methods. Due to limitations of the +//! RPC framework, there's currently no direct way to pass in state to RPC +//! methods without globals. +extern interfaces::Chain* g_rpc_chain; + void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers); /** @@ -34,6 +41,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques std::string HelpRequiringPassphrase(const CWallet*); void EnsureWalletIsUnlocked(const CWallet*); bool EnsureWalletIsAvailable(const CWallet*, bool avoidException); +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet); UniValue getaddressinfo(const JSONRPCRequest& request); UniValue signrawtransactionwithwallet(const JSONRPCRequest& request); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp new file mode 100644 index 0000000000..3eaaf3786c --- /dev/null +++ b/src/wallet/scriptpubkeyman.cpp @@ -0,0 +1,1405 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key_io.h> +#include <outputtype.h> +#include <script/descriptor.h> +#include <util/bip32.h> +#include <util/strencodings.h> +#include <util/translation.h> +#include <wallet/scriptpubkeyman.h> +#include <wallet/wallet.h> + +bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +{ + error.clear(); + TopUp(); + + // Generate a new key that is added to wallet + CPubKey new_key; + if (!GetKeyFromPool(new_key)) { + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + LearnRelatedScripts(new_key, type); + dest = GetDestinationForKey(new_key, type); + return true; +} + +typedef std::vector<unsigned char> valtype; + +namespace { + +/** + * This is an enum that tracks the execution context of a script, similar to + * SigVersion in script/interpreter. It is separate however because we want to + * distinguish between top-level scriptPubKey execution and P2SH redeemScript + * execution (a distinction that has no impact on consensus rules). + */ +enum class IsMineSigVersion +{ + TOP = 0, //!< scriptPubKey execution + P2SH = 1, //!< P2SH redeemScript + WITNESS_V0 = 2, //!< P2WSH witness script execution +}; + +/** + * This is an internal representation of isminetype + invalidity. + * Its order is significant, as we return the max of all explored + * possibilities. + */ +enum class IsMineResult +{ + NO = 0, //!< Not ours + WATCH_ONLY = 1, //!< Included in watch-only balance + SPENDABLE = 2, //!< Included in all balances + INVALID = 3, //!< Not spendable by anyone (uncompressed pubkey in segwit, P2SH inside P2SH or witness, witness inside witness) +}; + +bool PermitsUncompressed(IsMineSigVersion sigversion) +{ + return sigversion == IsMineSigVersion::TOP || sigversion == IsMineSigVersion::P2SH; +} + +bool HaveKeys(const std::vector<valtype>& pubkeys, const LegacyScriptPubKeyMan& keystore) +{ + for (const valtype& pubkey : pubkeys) { + CKeyID keyID = CPubKey(pubkey).GetID(); + if (!keystore.HaveKey(keyID)) return false; + } + return true; +} + +IsMineResult IsMineInner(const LegacyScriptPubKeyMan& keystore, const CScript& scriptPubKey, IsMineSigVersion sigversion) +{ + IsMineResult ret = IsMineResult::NO; + + std::vector<valtype> vSolutions; + txnouttype whichType = Solver(scriptPubKey, vSolutions); + + CKeyID keyID; + switch (whichType) + { + case TX_NONSTANDARD: + case TX_NULL_DATA: + case TX_WITNESS_UNKNOWN: + break; + case TX_PUBKEY: + keyID = CPubKey(vSolutions[0]).GetID(); + if (!PermitsUncompressed(sigversion) && vSolutions[0].size() != 33) { + return IsMineResult::INVALID; + } + if (keystore.HaveKey(keyID)) { + ret = std::max(ret, IsMineResult::SPENDABLE); + } + break; + case TX_WITNESS_V0_KEYHASH: + { + if (sigversion == IsMineSigVersion::WITNESS_V0) { + // P2WPKH inside P2WSH is invalid. + return IsMineResult::INVALID; + } + if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { + // We do not support bare witness outputs unless the P2SH version of it would be + // acceptable as well. This protects against matching before segwit activates. + // This also applies to the P2WSH case. + break; + } + ret = std::max(ret, IsMineInner(keystore, GetScriptForDestination(PKHash(uint160(vSolutions[0]))), IsMineSigVersion::WITNESS_V0)); + break; + } + case TX_PUBKEYHASH: + keyID = CKeyID(uint160(vSolutions[0])); + if (!PermitsUncompressed(sigversion)) { + CPubKey pubkey; + if (keystore.GetPubKey(keyID, pubkey) && !pubkey.IsCompressed()) { + return IsMineResult::INVALID; + } + } + if (keystore.HaveKey(keyID)) { + ret = std::max(ret, IsMineResult::SPENDABLE); + } + break; + case TX_SCRIPTHASH: + { + if (sigversion != IsMineSigVersion::TOP) { + // P2SH inside P2WSH or P2SH is invalid. + return IsMineResult::INVALID; + } + CScriptID scriptID = CScriptID(uint160(vSolutions[0])); + CScript subscript; + if (keystore.GetCScript(scriptID, subscript)) { + ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::P2SH)); + } + break; + } + case TX_WITNESS_V0_SCRIPTHASH: + { + if (sigversion == IsMineSigVersion::WITNESS_V0) { + // P2WSH inside P2WSH is invalid. + return IsMineResult::INVALID; + } + if (sigversion == IsMineSigVersion::TOP && !keystore.HaveCScript(CScriptID(CScript() << OP_0 << vSolutions[0]))) { + break; + } + uint160 hash; + CRIPEMD160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(hash.begin()); + CScriptID scriptID = CScriptID(hash); + CScript subscript; + if (keystore.GetCScript(scriptID, subscript)) { + ret = std::max(ret, IsMineInner(keystore, subscript, IsMineSigVersion::WITNESS_V0)); + } + break; + } + + case TX_MULTISIG: + { + // Never treat bare multisig outputs as ours (they can still be made watchonly-though) + if (sigversion == IsMineSigVersion::TOP) { + break; + } + + // Only consider transactions "mine" if we own ALL the + // keys involved. Multi-signature transactions that are + // partially owned (somebody else has a key that can spend + // them) enable spend-out-from-under-you attacks, especially + // in shared-wallet situations. + std::vector<valtype> keys(vSolutions.begin()+1, vSolutions.begin()+vSolutions.size()-1); + if (!PermitsUncompressed(sigversion)) { + for (size_t i = 0; i < keys.size(); i++) { + if (keys[i].size() != 33) { + return IsMineResult::INVALID; + } + } + } + if (HaveKeys(keys, keystore)) { + ret = std::max(ret, IsMineResult::SPENDABLE); + } + break; + } + } + + if (ret == IsMineResult::NO && keystore.HaveWatchOnly(scriptPubKey)) { + ret = std::max(ret, IsMineResult::WATCH_ONLY); + } + return ret; +} + +} // namespace + +isminetype LegacyScriptPubKeyMan::IsMine(const CScript& script) const +{ + switch (IsMineInner(*this, script, IsMineSigVersion::TOP)) { + case IsMineResult::INVALID: + case IsMineResult::NO: + return ISMINE_NO; + case IsMineResult::WATCH_ONLY: + return ISMINE_WATCH_ONLY; + case IsMineResult::SPENDABLE: + return ISMINE_SPENDABLE; + } + assert(false); +} + +bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) +{ + { + LOCK(cs_KeyStore); + if (!SetCrypted()) + return false; + + bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys + bool keyFail = false; + CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); + for (; mi != mapCryptedKeys.end(); ++mi) + { + const CPubKey &vchPubKey = (*mi).second.first; + const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; + CKey key; + if (!DecryptKey(vMasterKeyIn, vchCryptedSecret, vchPubKey, key)) + { + keyFail = true; + break; + } + keyPass = true; + if (fDecryptionThoroughlyChecked) + break; + } + if (keyPass && keyFail) + { + LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); + throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); + } + if (keyFail || (!keyPass && !accept_no_keys)) + return false; + vMasterKey = vMasterKeyIn; + fDecryptionThoroughlyChecked = true; + } + NotifyStatusChanged(this); + return true; +} + +bool LegacyScriptPubKeyMan::EncryptKeys(CKeyingMaterial& vMasterKeyIn) +{ + LOCK(cs_KeyStore); + if (!mapCryptedKeys.empty() || IsCrypted()) + return false; + + fUseCrypto = true; + for (const KeyMap::value_type& mKey : mapKeys) + { + const CKey &key = mKey.second; + CPubKey vchPubKey = key.GetPubKey(); + CKeyingMaterial vchSecret(key.begin(), key.end()); + std::vector<unsigned char> vchCryptedSecret; + if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) + return false; + if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) + return false; + } + mapKeys.clear(); + return true; +} + +bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) +{ + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { + return false; + } + return true; +} + +void LegacyScriptPubKeyMan::KeepDestination(int64_t index) +{ + KeepKey(index); +} + +void LegacyScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) +{ + ReturnKey(index, internal, pubkey); +} + +void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +{ + AssertLockHeld(cs_wallet); + // extract addresses and check if they match with an unused keypool key + for (const auto& keyid : GetAffectedKeys(script, *this)) { + std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); + if (mi != m_pool_key_to_index.end()) { + WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); + MarkReserveKeysAsUsed(mi->second); + + if (!TopUp()) { + WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); + } + } + } +} + +void LegacyScriptPubKeyMan::UpgradeKeyMetadata() +{ + AssertLockHeld(cs_wallet); + if (m_storage.IsLocked() || m_storage.IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { + return; + } + + std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(m_storage.GetDatabase()); + for (auto& meta_pair : mapKeyMetadata) { + CKeyMetadata& meta = meta_pair.second; + if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin + CKey key; + GetKey(meta.hd_seed_id, key); + CExtKey masterKey; + masterKey.SetSeed(key.begin(), key.size()); + // Add to map + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); + if (!ParseHDKeypath(meta.hdKeypath, meta.key_origin.path)) { + throw std::runtime_error("Invalid stored hdKeypath"); + } + meta.has_key_origin = true; + if (meta.nVersion < CKeyMetadata::VERSION_WITH_KEY_ORIGIN) { + meta.nVersion = CKeyMetadata::VERSION_WITH_KEY_ORIGIN; + } + + // Write meta to wallet + CPubKey pubkey; + if (GetPubKey(meta_pair.first, pubkey)) { + batch->WriteKeyMetadata(meta, pubkey, true); + } + } + } +} + +bool LegacyScriptPubKeyMan::SetupGeneration(bool force) +{ + if ((CanGenerateKeys() && !force) || m_storage.IsLocked()) { + return false; + } + + SetHDSeed(GenerateNewSeed()); + if (!NewKeyPool()) { + return false; + } + return true; +} + +bool LegacyScriptPubKeyMan::IsHDEnabled() const +{ + return !hdChain.seed_id.IsNull(); +} + +bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) +{ + LOCK(cs_wallet); + // Check if the keypool has keys + bool keypool_has_keys; + if (internal && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + keypool_has_keys = setInternalKeyPool.size() > 0; + } else { + keypool_has_keys = KeypoolCountExternalKeys() > 0; + } + // If the keypool doesn't have keys, check if we can generate them + if (!keypool_has_keys) { + return CanGenerateKeys(); + } + return keypool_has_keys; +} + +bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) +{ + AssertLockHeld(cs_wallet); + error = ""; + bool hd_upgrade = false; + bool split_upgrade = false; + if (m_storage.CanSupportFeature(FEATURE_HD) && !IsHDEnabled()) { + WalletLogPrintf("Upgrading wallet to HD\n"); + m_storage.SetMinVersion(FEATURE_HD); + + // generate a new master key + CPubKey masterPubKey = GenerateNewSeed(); + SetHDSeed(masterPubKey); + hd_upgrade = true; + } + // Upgrade to HD chain split if necessary + if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + WalletLogPrintf("Upgrading wallet to use HD chain split\n"); + m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); + split_upgrade = FEATURE_HD_SPLIT > prev_version; + } + // Mark all keys currently in the keypool as pre-split + if (split_upgrade) { + MarkPreSplitKeys(); + } + // Regenerate the keypool if upgraded to HD + if (hd_upgrade) { + if (!TopUp()) { + error = _("Unable to generate keys").translated; + return false; + } + } + return true; +} + +bool LegacyScriptPubKeyMan::HavePrivateKeys() const +{ + LOCK(cs_KeyStore); + return !mapKeys.empty() || !mapCryptedKeys.empty(); +} + +void LegacyScriptPubKeyMan::RewriteDB() +{ + AssertLockHeld(cs_wallet); + setInternalKeyPool.clear(); + setExternalKeyPool.clear(); + m_pool_key_to_index.clear(); + // Note: can't top-up keypool here, because wallet is locked. + // User will be prompted to unlock wallet the next operation + // that requires a new key. +} + +static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, WalletBatch& batch) { + if (setKeyPool.empty()) { + return GetTime(); + } + + CKeyPool keypool; + int64_t nIndex = *(setKeyPool.begin()); + if (!batch.ReadPool(nIndex, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); + } + assert(keypool.vchPubKey.IsValid()); + return keypool.nTime; +} + +int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() +{ + LOCK(cs_wallet); + + WalletBatch batch(m_storage.GetDatabase()); + + // load oldest key from keypool, get time and return + int64_t oldestKey = GetOldestKeyTimeInPool(setExternalKeyPool, batch); + if (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + oldestKey = std::max(GetOldestKeyTimeInPool(setInternalKeyPool, batch), oldestKey); + if (!set_pre_split_keypool.empty()) { + oldestKey = std::max(GetOldestKeyTimeInPool(set_pre_split_keypool, batch), oldestKey); + } + } + + return oldestKey; +} + +size_t LegacyScriptPubKeyMan::KeypoolCountExternalKeys() +{ + AssertLockHeld(cs_wallet); + return setExternalKeyPool.size() + set_pre_split_keypool.size(); +} + +unsigned int LegacyScriptPubKeyMan::GetKeyPoolSize() const +{ + AssertLockHeld(cs_wallet); + return setInternalKeyPool.size() + setExternalKeyPool.size(); +} + +int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const +{ + AssertLockHeld(cs_wallet); + return nTimeFirstKey; +} + +const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +{ + AssertLockHeld(cs_wallet); + + CKeyID key_id = GetKeyForDestination(*this, dest); + if (!key_id.IsNull()) { + auto it = mapKeyMetadata.find(key_id); + if (it != mapKeyMetadata.end()) { + return &it->second; + } + } + + CScript scriptPubKey = GetScriptForDestination(dest); + auto it = m_script_metadata.find(CScriptID(scriptPubKey)); + if (it != m_script_metadata.end()) { + return &it->second; + } + + return nullptr; +} + +/** + * Update wallet first key creation time. This should be called whenever keys + * are added to the wallet, with the oldest key creation time. + */ +void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) +{ + AssertLockHeld(cs_wallet); + if (nCreateTime <= 1) { + // Cannot determine birthday information, so set the wallet birthday to + // the beginning of time. + nTimeFirstKey = 1; + } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { + nTimeFirstKey = nCreateTime; + } +} + +bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) +{ + return AddKeyPubKeyInner(key, pubkey); +} + +bool LegacyScriptPubKeyMan::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) +{ + WalletBatch batch(m_storage.GetDatabase()); + return LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(batch, secret, pubkey); +} + +bool LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) +{ + AssertLockHeld(cs_wallet); + + // Make sure we aren't adding private keys to private key disabled wallets + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + + // FillableSigningProvider has no concept of wallet databases, but calls AddCryptedKey + // which is overridden below. To avoid flushes, the database handle is + // tunneled through to it. + bool needsDB = !encrypted_batch; + if (needsDB) { + encrypted_batch = &batch; + } + if (!AddKeyPubKeyInner(secret, pubkey)) { + if (needsDB) encrypted_batch = nullptr; + return false; + } + if (needsDB) encrypted_batch = nullptr; + + // check if we need to remove from watch-only + CScript script; + script = GetScriptForDestination(PKHash(pubkey)); + if (HaveWatchOnly(script)) { + RemoveWatchOnly(script); + } + script = GetScriptForRawPubKey(pubkey); + if (HaveWatchOnly(script)) { + RemoveWatchOnly(script); + } + + if (!IsCrypted()) { + return batch.WriteKey(pubkey, + secret.GetPrivKey(), + mapKeyMetadata[pubkey.GetID()]); + } + m_storage.UnsetBlankWalletFlag(batch); + return true; +} + +bool LegacyScriptPubKeyMan::LoadCScript(const CScript& redeemScript) +{ + /* A sanity check was added in pull #3843 to avoid adding redeemScripts + * that never can be redeemed. However, old wallets may still contain + * these. Do not add them to the wallet and warn. */ + if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) + { + std::string strAddr = EncodeDestination(ScriptHash(redeemScript)); + WalletLogPrintf("%s: Warning: This wallet contains a redeemScript of size %i which exceeds maximum size %i thus can never be redeemed. Do not use address %s.\n", __func__, redeemScript.size(), MAX_SCRIPT_ELEMENT_SIZE, strAddr); + return true; + } + + return FillableSigningProvider::AddCScript(redeemScript); +} + +void LegacyScriptPubKeyMan::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) +{ + AssertLockHeld(cs_wallet); + UpdateTimeFirstKey(meta.nCreateTime); + mapKeyMetadata[keyID] = meta; +} + +void LegacyScriptPubKeyMan::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) +{ + AssertLockHeld(cs_wallet); + UpdateTimeFirstKey(meta.nCreateTime); + m_script_metadata[script_id] = meta; +} + +bool LegacyScriptPubKeyMan::AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey) +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return FillableSigningProvider::AddKeyPubKey(key, pubkey); + } + + if (m_storage.IsLocked()) { + return false; + } + + std::vector<unsigned char> vchCryptedSecret; + CKeyingMaterial vchSecret(key.begin(), key.end()); + if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) { + return false; + } + + if (!AddCryptedKey(pubkey, vchCryptedSecret)) { + return false; + } + return true; +} + +bool LegacyScriptPubKeyMan::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +{ + return AddCryptedKeyInner(vchPubKey, vchCryptedSecret); +} + +bool LegacyScriptPubKeyMan::AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) +{ + LOCK(cs_KeyStore); + if (!SetCrypted()) { + return false; + } + + mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); + ImplicitlyLearnRelatedKeyScripts(vchPubKey); + return true; +} + +bool LegacyScriptPubKeyMan::AddCryptedKey(const CPubKey &vchPubKey, + const std::vector<unsigned char> &vchCryptedSecret) +{ + if (!AddCryptedKeyInner(vchPubKey, vchCryptedSecret)) + return false; + { + LOCK(cs_wallet); + if (encrypted_batch) + return encrypted_batch->WriteCryptedKey(vchPubKey, + vchCryptedSecret, + mapKeyMetadata[vchPubKey.GetID()]); + else + return WalletBatch(m_storage.GetDatabase()).WriteCryptedKey(vchPubKey, + vchCryptedSecret, + mapKeyMetadata[vchPubKey.GetID()]); + } +} + +bool LegacyScriptPubKeyMan::HaveWatchOnly(const CScript &dest) const +{ + LOCK(cs_KeyStore); + return setWatchOnly.count(dest) > 0; +} + +bool LegacyScriptPubKeyMan::HaveWatchOnly() const +{ + LOCK(cs_KeyStore); + return (!setWatchOnly.empty()); +} + +static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) +{ + std::vector<std::vector<unsigned char>> solutions; + return Solver(dest, solutions) == TX_PUBKEY && + (pubKeyOut = CPubKey(solutions[0])).IsFullyValid(); +} + +bool LegacyScriptPubKeyMan::RemoveWatchOnly(const CScript &dest) +{ + AssertLockHeld(cs_wallet); + { + LOCK(cs_KeyStore); + setWatchOnly.erase(dest); + CPubKey pubKey; + if (ExtractPubKey(dest, pubKey)) { + mapWatchKeys.erase(pubKey.GetID()); + } + // Related CScripts are not removed; having superfluous scripts around is + // harmless (see comment in ImplicitlyLearnRelatedKeyScripts). + } + + if (!HaveWatchOnly()) + NotifyWatchonlyChanged(false); + if (!WalletBatch(m_storage.GetDatabase()).EraseWatchOnly(dest)) + return false; + + return true; +} + +bool LegacyScriptPubKeyMan::LoadWatchOnly(const CScript &dest) +{ + return AddWatchOnlyInMem(dest); +} + +bool LegacyScriptPubKeyMan::AddWatchOnlyInMem(const CScript &dest) +{ + LOCK(cs_KeyStore); + setWatchOnly.insert(dest); + CPubKey pubKey; + if (ExtractPubKey(dest, pubKey)) { + mapWatchKeys[pubKey.GetID()] = pubKey; + ImplicitlyLearnRelatedKeyScripts(pubKey); + } + return true; +} + +bool LegacyScriptPubKeyMan::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) +{ + if (!AddWatchOnlyInMem(dest)) + return false; + const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; + UpdateTimeFirstKey(meta.nCreateTime); + NotifyWatchonlyChanged(true); + if (batch.WriteWatchOnly(dest, meta)) { + m_storage.UnsetBlankWalletFlag(batch); + return true; + } + return false; +} + +bool LegacyScriptPubKeyMan::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) +{ + m_script_metadata[CScriptID(dest)].nCreateTime = create_time; + return AddWatchOnlyWithDB(batch, dest); +} + +bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest) +{ + WalletBatch batch(m_storage.GetDatabase()); + return AddWatchOnlyWithDB(batch, dest); +} + +bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTime) +{ + m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; + return AddWatchOnly(dest); +} + +void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) +{ + LOCK(cs_wallet); + if (!memonly && !WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) + throw std::runtime_error(std::string(__func__) + ": writing chain failed"); + + hdChain = chain; +} + +bool LegacyScriptPubKeyMan::HaveKey(const CKeyID &address) const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return FillableSigningProvider::HaveKey(address); + } + return mapCryptedKeys.count(address) > 0; +} + +bool LegacyScriptPubKeyMan::GetKey(const CKeyID &address, CKey& keyOut) const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return FillableSigningProvider::GetKey(address, keyOut); + } + + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) + { + const CPubKey &vchPubKey = (*mi).second.first; + const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; + return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); + } + return false; +} + +bool LegacyScriptPubKeyMan::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const +{ + CKeyMetadata meta; + { + LOCK(cs_wallet); + auto it = mapKeyMetadata.find(keyID); + if (it != mapKeyMetadata.end()) { + meta = it->second; + } + } + if (meta.has_key_origin) { + std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); + info.path = meta.key_origin.path; + } else { // Single pubkeys get the master fingerprint of themselves + std::copy(keyID.begin(), keyID.begin() + 4, info.fingerprint); + } + return true; +} + +bool LegacyScriptPubKeyMan::GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const +{ + LOCK(cs_KeyStore); + WatchKeyMap::const_iterator it = mapWatchKeys.find(address); + if (it != mapWatchKeys.end()) { + pubkey_out = it->second; + return true; + } + return false; +} + +bool LegacyScriptPubKeyMan::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + if (!FillableSigningProvider::GetPubKey(address, vchPubKeyOut)) { + return GetWatchPubKey(address, vchPubKeyOut); + } + return true; + } + + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) + { + vchPubKeyOut = (*mi).second.first; + return true; + } + // Check for watch-only pubkeys + return GetWatchPubKey(address, vchPubKeyOut); +} + +CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) +{ + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); + AssertLockHeld(cs_wallet); + bool fCompressed = m_storage.CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets + + CKey secret; + + // Create new metadata + int64_t nCreationTime = GetTime(); + CKeyMetadata metadata(nCreationTime); + + // use HD key derivation if HD was enabled during wallet creation and a seed is present + if (IsHDEnabled()) { + DeriveNewChildKey(batch, metadata, secret, (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); + } else { + secret.MakeNewKey(fCompressed); + } + + // Compressed public keys were introduced in version 0.6.0 + if (fCompressed) { + m_storage.SetMinVersion(FEATURE_COMPRPUBKEY); + } + + CPubKey pubkey = secret.GetPubKey(); + assert(secret.VerifyPubKey(pubkey)); + + mapKeyMetadata[pubkey.GetID()] = metadata; + UpdateTimeFirstKey(nCreationTime); + + if (!AddKeyPubKeyWithDB(batch, secret, pubkey)) { + throw std::runtime_error(std::string(__func__) + ": AddKey failed"); + } + return pubkey; +} + +const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; + +void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal) +{ + // for now we use a fixed keypath scheme of m/0'/0'/k + CKey seed; //seed (256bit) + CExtKey masterKey; //hd master key + CExtKey accountKey; //key at m/0' + CExtKey chainChildKey; //key at m/0'/0' (external) or m/0'/1' (internal) + CExtKey childKey; //key at m/0'/0'/<n>' + + // try to get the seed + if (!GetKey(hdChain.seed_id, seed)) + throw std::runtime_error(std::string(__func__) + ": seed not found"); + + masterKey.SetSeed(seed.begin(), seed.size()); + + // derive m/0' + // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) + masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); + + // derive m/0'/0' (external chain) OR m/0'/1' (internal chain) + assert(internal ? m_storage.CanSupportFeature(FEATURE_HD_SPLIT) : true); + accountKey.Derive(chainChildKey, BIP32_HARDENED_KEY_LIMIT+(internal ? 1 : 0)); + + // derive child key at next index, skip keys already known to the wallet + do { + // always derive hardened keys + // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range + // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 + if (internal) { + chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hdChain.nInternalChainCounter++; + } + else { + chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); + hdChain.nExternalChainCounter++; + } + } while (HaveKey(childKey.key.GetPubKey().GetID())); + secret = childKey.key; + metadata.hd_seed_id = hdChain.seed_id; + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); + metadata.has_key_origin = true; + // update the chain model in the database + if (!batch.WriteHDChain(hdChain)) + throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); +} + +void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) +{ + AssertLockHeld(cs_wallet); + if (keypool.m_pre_split) { + set_pre_split_keypool.insert(nIndex); + } else if (keypool.fInternal) { + setInternalKeyPool.insert(nIndex); + } else { + setExternalKeyPool.insert(nIndex); + } + m_max_keypool_index = std::max(m_max_keypool_index, nIndex); + m_pool_key_to_index[keypool.vchPubKey.GetID()] = nIndex; + + // If no metadata exists yet, create a default with the pool key's + // creation time. Note that this may be overwritten by actually + // stored metadata for that key later, which is fine. + CKeyID keyid = keypool.vchPubKey.GetID(); + if (mapKeyMetadata.count(keyid) == 0) + mapKeyMetadata[keyid] = CKeyMetadata(keypool.nTime); +} + +bool LegacyScriptPubKeyMan::CanGenerateKeys() +{ + // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) + LOCK(cs_wallet); + return IsHDEnabled() || !m_storage.CanSupportFeature(FEATURE_HD); +} + +CPubKey LegacyScriptPubKeyMan::GenerateNewSeed() +{ + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + CKey key; + key.MakeNewKey(true); + return DeriveNewSeed(key); +} + +CPubKey LegacyScriptPubKeyMan::DeriveNewSeed(const CKey& key) +{ + int64_t nCreationTime = GetTime(); + CKeyMetadata metadata(nCreationTime); + + // calculate the seed + CPubKey seed = key.GetPubKey(); + assert(key.VerifyPubKey(seed)); + + // set the hd keypath to "s" -> Seed, refers the seed to itself + metadata.hdKeypath = "s"; + metadata.has_key_origin = false; + metadata.hd_seed_id = seed.GetID(); + + { + LOCK(cs_wallet); + + // mem store the metadata + mapKeyMetadata[seed.GetID()] = metadata; + + // write the key&metadata to the database + if (!AddKeyPubKey(key, seed)) + throw std::runtime_error(std::string(__func__) + ": AddKeyPubKey failed"); + } + + return seed; +} + +void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) +{ + LOCK(cs_wallet); + // store the keyid (hash160) together with + // the child index counter in the database + // as a hdchain object + CHDChain newHdChain; + newHdChain.nVersion = m_storage.CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; + newHdChain.seed_id = seed.GetID(); + SetHDChain(newHdChain, false); + NotifyCanGetAddressesChanged(); + WalletBatch batch(m_storage.GetDatabase()); + m_storage.UnsetBlankWalletFlag(batch); +} + +/** + * Mark old keypool keys as used, + * and generate all new keys + */ +bool LegacyScriptPubKeyMan::NewKeyPool() +{ + if (m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + return false; + } + { + LOCK(cs_wallet); + WalletBatch batch(m_storage.GetDatabase()); + + for (const int64_t nIndex : setInternalKeyPool) { + batch.ErasePool(nIndex); + } + setInternalKeyPool.clear(); + + for (const int64_t nIndex : setExternalKeyPool) { + batch.ErasePool(nIndex); + } + setExternalKeyPool.clear(); + + for (const int64_t nIndex : set_pre_split_keypool) { + batch.ErasePool(nIndex); + } + set_pre_split_keypool.clear(); + + m_pool_key_to_index.clear(); + + if (!TopUp()) { + return false; + } + WalletLogPrintf("LegacyScriptPubKeyMan::NewKeyPool rewrote keypool\n"); + } + return true; +} + +bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) +{ + if (!CanGenerateKeys()) { + return false; + } + { + LOCK(cs_wallet); + + if (m_storage.IsLocked()) return false; + + // Top up key pool + unsigned int nTargetSize; + if (kpSize > 0) + nTargetSize = kpSize; + else + nTargetSize = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); + + // count amount of available keys (internal, external) + // make sure the keypool of external and internal keys fits the user selected target (-keypool) + int64_t missingExternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setExternalKeyPool.size(), (int64_t) 0); + int64_t missingInternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setInternalKeyPool.size(), (int64_t) 0); + + if (!IsHDEnabled() || !m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) + { + // don't create extra internal keys + missingInternal = 0; + } + bool internal = false; + WalletBatch batch(m_storage.GetDatabase()); + for (int64_t i = missingInternal + missingExternal; i--;) + { + if (i < missingInternal) { + internal = true; + } + + CPubKey pubkey(GenerateNewKey(batch, internal)); + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + } + if (missingInternal + missingExternal > 0) { + WalletLogPrintf("keypool added %d keys (%d internal), size=%u (%u internal)\n", missingInternal + missingExternal, missingInternal, setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(), setInternalKeyPool.size()); + } + } + NotifyCanGetAddressesChanged(); + return true; +} + +void LegacyScriptPubKeyMan::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) +{ + LOCK(cs_wallet); + assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? + int64_t index = ++m_max_keypool_index; + if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { + throw std::runtime_error(std::string(__func__) + ": writing imported pubkey failed"); + } + if (internal) { + setInternalKeyPool.insert(index); + } else { + setExternalKeyPool.insert(index); + } + m_pool_key_to_index[pubkey.GetID()] = index; +} + +void LegacyScriptPubKeyMan::KeepKey(int64_t nIndex) +{ + // Remove from key pool + WalletBatch batch(m_storage.GetDatabase()); + batch.ErasePool(nIndex); + WalletLogPrintf("keypool keep %d\n", nIndex); +} + +void LegacyScriptPubKeyMan::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey) +{ + // Return to key pool + { + LOCK(cs_wallet); + if (fInternal) { + setInternalKeyPool.insert(nIndex); + } else if (!set_pre_split_keypool.empty()) { + set_pre_split_keypool.insert(nIndex); + } else { + setExternalKeyPool.insert(nIndex); + } + m_pool_key_to_index[pubkey.GetID()] = nIndex; + NotifyCanGetAddressesChanged(); + } + WalletLogPrintf("keypool return %d\n", nIndex); +} + +bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, bool internal) +{ + if (!CanGetAddresses(internal)) { + return false; + } + + CKeyPool keypool; + { + LOCK(cs_wallet); + int64_t nIndex; + if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (m_storage.IsLocked()) return false; + WalletBatch batch(m_storage.GetDatabase()); + result = GenerateNewKey(batch, internal); + return true; + } + KeepKey(nIndex); + result = keypool.vchPubKey; + } + return true; +} + +bool LegacyScriptPubKeyMan::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal) +{ + nIndex = -1; + keypool.vchPubKey = CPubKey(); + { + LOCK(cs_wallet); + + TopUp(); + + bool fReturningInternal = fRequestedInternal; + fReturningInternal &= (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) || m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + bool use_split_keypool = set_pre_split_keypool.empty(); + std::set<int64_t>& setKeyPool = use_split_keypool ? (fReturningInternal ? setInternalKeyPool : setExternalKeyPool) : set_pre_split_keypool; + + // Get the oldest key + if (setKeyPool.empty()) { + return false; + } + + WalletBatch batch(m_storage.GetDatabase()); + + auto it = setKeyPool.begin(); + nIndex = *it; + setKeyPool.erase(it); + if (!batch.ReadPool(nIndex, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read failed"); + } + CPubKey pk; + if (!GetPubKey(keypool.vchPubKey.GetID(), pk)) { + throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); + } + // If the key was pre-split keypool, we don't care about what type it is + if (use_split_keypool && keypool.fInternal != fReturningInternal) { + throw std::runtime_error(std::string(__func__) + ": keypool entry misclassified"); + } + if (!keypool.vchPubKey.IsValid()) { + throw std::runtime_error(std::string(__func__) + ": keypool entry invalid"); + } + + m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); + WalletLogPrintf("keypool reserve %d\n", nIndex); + } + NotifyCanGetAddressesChanged(); + return true; +} + +void LegacyScriptPubKeyMan::LearnRelatedScripts(const CPubKey& key, OutputType type) +{ + if (key.IsCompressed() && (type == OutputType::P2SH_SEGWIT || type == OutputType::BECH32)) { + CTxDestination witdest = WitnessV0KeyHash(key.GetID()); + CScript witprog = GetScriptForDestination(witdest); + // Make sure the resulting program is solvable. + assert(IsSolvable(*this, witprog)); + AddCScript(witprog); + } +} + +void LegacyScriptPubKeyMan::LearnAllRelatedScripts(const CPubKey& key) +{ + // OutputType::P2SH_SEGWIT always adds all necessary scripts for all types. + LearnRelatedScripts(key, OutputType::P2SH_SEGWIT); +} + +void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) +{ + AssertLockHeld(cs_wallet); + bool internal = setInternalKeyPool.count(keypool_id); + if (!internal) assert(setExternalKeyPool.count(keypool_id) || set_pre_split_keypool.count(keypool_id)); + std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); + auto it = setKeyPool->begin(); + + WalletBatch batch(m_storage.GetDatabase()); + while (it != std::end(*setKeyPool)) { + const int64_t& index = *(it); + if (index > keypool_id) break; // set*KeyPool is ordered + + CKeyPool keypool; + if (batch.ReadPool(index, keypool)) { //TODO: This should be unnecessary + m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); + } + LearnAllRelatedScripts(keypool.vchPubKey); + batch.ErasePool(index); + WalletLogPrintf("keypool index %d removed\n", index); + it = setKeyPool->erase(it); + } +} + +std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider) +{ + std::vector<CScript> dummy; + FlatSigningProvider out; + InferDescriptor(spk, provider)->Expand(0, DUMMY_SIGNING_PROVIDER, dummy, out); + std::vector<CKeyID> ret; + for (const auto& entry : out.pubkeys) { + ret.push_back(entry.first); + } + return ret; +} + +void LegacyScriptPubKeyMan::MarkPreSplitKeys() +{ + WalletBatch batch(m_storage.GetDatabase()); + for (auto it = setExternalKeyPool.begin(); it != setExternalKeyPool.end();) { + int64_t index = *it; + CKeyPool keypool; + if (!batch.ReadPool(index, keypool)) { + throw std::runtime_error(std::string(__func__) + ": read keypool entry failed"); + } + keypool.m_pre_split = true; + if (!batch.WritePool(index, keypool)) { + throw std::runtime_error(std::string(__func__) + ": writing modified keypool entry failed"); + } + set_pre_split_keypool.insert(index); + it = setExternalKeyPool.erase(it); + } +} + +bool LegacyScriptPubKeyMan::AddCScript(const CScript& redeemScript) +{ + WalletBatch batch(m_storage.GetDatabase()); + return AddCScriptWithDB(batch, redeemScript); +} + +bool LegacyScriptPubKeyMan::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript) +{ + if (!FillableSigningProvider::AddCScript(redeemScript)) + return false; + if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) { + m_storage.UnsetBlankWalletFlag(batch); + return true; + } + return false; +} + +bool LegacyScriptPubKeyMan::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) +{ + LOCK(cs_wallet); + std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); + mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; + mapKeyMetadata[pubkey.GetID()].has_key_origin = true; + mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); + return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); +} + +bool LegacyScriptPubKeyMan::ImportScripts(const std::set<CScript> scripts, int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const auto& entry : scripts) { + CScriptID id(entry); + if (HaveCScript(id)) { + WalletLogPrintf("Already have script %s, skipping\n", HexStr(entry)); + continue; + } + if (!AddCScriptWithDB(batch, entry)) { + return false; + } + + if (timestamp > 0) { + m_script_metadata[CScriptID(entry)].nCreateTime = timestamp; + } + } + if (timestamp > 0) { + UpdateTimeFirstKey(timestamp); + } + + return true; +} + +bool LegacyScriptPubKeyMan::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const auto& entry : privkey_map) { + const CKey& key = entry.second; + CPubKey pubkey = key.GetPubKey(); + const CKeyID& id = entry.first; + assert(key.VerifyPubKey(pubkey)); + // Skip if we already have the key + if (HaveKey(id)) { + WalletLogPrintf("Already have key with pubkey %s, skipping\n", HexStr(pubkey)); + continue; + } + mapKeyMetadata[id].nCreateTime = timestamp; + // If the private key is not present in the wallet, insert it. + if (!AddKeyPubKeyWithDB(batch, key, pubkey)) { + return false; + } + UpdateTimeFirstKey(timestamp); + } + return true; +} + +bool LegacyScriptPubKeyMan::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const auto& entry : key_origins) { + AddKeyOriginWithDB(batch, entry.second.first, entry.second.second); + } + for (const CKeyID& id : ordered_pubkeys) { + auto entry = pubkey_map.find(id); + if (entry == pubkey_map.end()) { + continue; + } + const CPubKey& pubkey = entry->second; + CPubKey temp; + if (GetPubKey(id, temp)) { + // Already have pubkey, skipping + WalletLogPrintf("Already have pubkey %s, skipping\n", HexStr(temp)); + continue; + } + if (!AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) { + return false; + } + mapKeyMetadata[id].nCreateTime = timestamp; + + // Add to keypool only works with pubkeys + if (add_keypool) { + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + NotifyCanGetAddressesChanged(); + } + } + return true; +} + +bool LegacyScriptPubKeyMan::ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) +{ + WalletBatch batch(m_storage.GetDatabase()); + for (const CScript& script : script_pub_keys) { + if (!have_solving_data || !IsMine(script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated + if (!AddWatchOnlyWithDB(batch, script, timestamp)) { + return false; + } + } + } + return true; +} + +std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return FillableSigningProvider::GetKeys(); + } + std::set<CKeyID> set_address; + for (const auto& mi : mapCryptedKeys) { + set_address.insert(mi.first); + } + return set_address; +} + +// Temporary CWallet accessors and aliases. +LegacyScriptPubKeyMan::LegacyScriptPubKeyMan(CWallet& wallet) + : ScriptPubKeyMan(wallet), + m_wallet(wallet), + cs_wallet(wallet.cs_wallet), + vMasterKey(wallet.vMasterKey), + fUseCrypto(wallet.fUseCrypto), + fDecryptionThoroughlyChecked(wallet.fDecryptionThoroughlyChecked) {} + +bool LegacyScriptPubKeyMan::SetCrypted() { return m_wallet.SetCrypted(); } +bool LegacyScriptPubKeyMan::IsCrypted() const { return m_wallet.IsCrypted(); } +void LegacyScriptPubKeyMan::NotifyWatchonlyChanged(bool fHaveWatchOnly) const { return m_wallet.NotifyWatchonlyChanged(fHaveWatchOnly); } +void LegacyScriptPubKeyMan::NotifyCanGetAddressesChanged() const { return m_wallet.NotifyCanGetAddressesChanged(); } +template<typename... Params> void LegacyScriptPubKeyMan::WalletLogPrintf(const std::string& fmt, const Params&... parameters) const { return m_wallet.WalletLogPrintf(fmt, parameters...); } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h new file mode 100644 index 0000000000..4f17156792 --- /dev/null +++ b/src/wallet/scriptpubkeyman.h @@ -0,0 +1,419 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_SCRIPTPUBKEYMAN_H +#define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H + +#include <script/signingprovider.h> +#include <script/standard.h> +#include <wallet/crypter.h> +#include <wallet/ismine.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> + +#include <boost/signals2/signal.hpp> + +enum class OutputType; + +// Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. +// It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as +// wallet flags, wallet version, encryption keys, encryption status, and the database itself. This allows a +// ScriptPubKeyMan to have callbacks into CWallet without causing a circular dependency. +// WalletStorage should be the same for all ScriptPubKeyMans of a wallet. +class WalletStorage +{ +public: + virtual ~WalletStorage() = default; + virtual const std::string GetDisplayName() const = 0; + virtual WalletDatabase& GetDatabase() = 0; + virtual bool IsWalletFlagSet(uint64_t) const = 0; + virtual void UnsetBlankWalletFlag(WalletBatch&) = 0; + virtual bool CanSupportFeature(enum WalletFeature) const = 0; + virtual void SetMinVersion(enum WalletFeature, WalletBatch* = nullptr, bool = false) = 0; + virtual bool IsLocked() const = 0; +}; + +//! Default for -keypool +static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; + +std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider); + +/** A key from a CWallet's keypool + * + * The wallet holds one (for pre HD-split wallets) or several keypools. These + * are sets of keys that have not yet been used to provide addresses or receive + * change. + * + * The Bitcoin Core wallet was originally a collection of unrelated private + * keys with their associated addresses. If a non-HD wallet generated a + * key/address, gave that address out and then restored a backup from before + * that key's generation, then any funds sent to that address would be + * lost definitively. + * + * The keypool was implemented to avoid this scenario (commit: 10384941). The + * wallet would generate a set of keys (100 by default). When a new public key + * was required, either to give out as an address or to use in a change output, + * it would be drawn from the keypool. The keypool would then be topped up to + * maintain 100 keys. This ensured that as long as the wallet hadn't used more + * than 100 keys since the previous backup, all funds would be safe, since a + * restored wallet would be able to scan for all owned addresses. + * + * A keypool also allowed encrypted wallets to give out addresses without + * having to be decrypted to generate a new private key. + * + * With the introduction of HD wallets (commit: f1902510), the keypool + * essentially became an address look-ahead pool. Restoring old backups can no + * longer definitively lose funds as long as the addresses used were from the + * wallet's HD seed (since all private keys can be rederived from the seed). + * However, if many addresses were used since the backup, then the wallet may + * not know how far ahead in the HD chain to look for its addresses. The + * keypool is used to implement a 'gap limit'. The keypool maintains a set of + * keys (by default 1000) ahead of the last used key and scans for the + * addresses of those keys. This avoids the risk of not seeing transactions + * involving the wallet's addresses, or of re-using the same address. + * + * The HD-split wallet feature added a second keypool (commit: 02592f4c). There + * is an external keypool (for addresses to hand out) and an internal keypool + * (for change addresses). + * + * Keypool keys are stored in the wallet/keystore's keymap. The keypool data is + * stored as sets of indexes in the wallet (setInternalKeyPool, + * setExternalKeyPool and set_pre_split_keypool), and a map from the key to the + * index (m_pool_key_to_index). The CKeyPool object is used to + * serialize/deserialize the pool data to/from the database. + */ +class CKeyPool +{ +public: + //! The time at which the key was generated. Set in AddKeypoolPubKeyWithDB + int64_t nTime; + //! The public key + CPubKey vchPubKey; + //! Whether this keypool entry is in the internal keypool (for change outputs) + bool fInternal; + //! Whether this key was generated for a keypool before the wallet was upgraded to HD-split + bool m_pre_split; + + CKeyPool(); + CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) + READWRITE(nVersion); + READWRITE(nTime); + READWRITE(vchPubKey); + if (ser_action.ForRead()) { + try { + READWRITE(fInternal); + } + catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; + } + try { + READWRITE(m_pre_split); + } + catch (std::ios_base::failure&) { + /* flag as postsplit address if we can't read the m_pre_split boolean + (this will be the case for any wallet that upgrades to HD chain split)*/ + m_pre_split = false; + } + } + else { + READWRITE(fInternal); + READWRITE(m_pre_split); + } + } +}; + +/* + * A class implementing ScriptPubKeyMan manages some (or all) scriptPubKeys used in a wallet. + * It contains the scripts and keys related to the scriptPubKeys it manages. + * A ScriptPubKeyMan will be able to give out scriptPubKeys to be used, as well as marking + * when a scriptPubKey has been used. It also handles when and how to store a scriptPubKey + * and its related scripts and keys, including encryption. + */ +class ScriptPubKeyMan +{ +protected: + WalletStorage& m_storage; + +public: + ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} + virtual ~ScriptPubKeyMan() {}; + virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { return false; } + virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } + + virtual bool GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) { return false; } + virtual void KeepDestination(int64_t index) {} + virtual void ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) {} + + virtual bool TopUp(unsigned int size = 0) { return false; } + + //! Mark unused addresses as being used + virtual void MarkUnusedAddresses(const CScript& script) {} + + /** Sets up the key generation stuff, i.e. generates new HD seeds and sets them as active. + * Returns false if already setup or setup fails, true if setup is successful + * Set force=true to make it re-setup if already setup, used for upgrades + */ + virtual bool SetupGeneration(bool force = false) { return false; } + + /* Returns true if HD is enabled */ + virtual bool IsHDEnabled() const { return false; } + + /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ + virtual bool CanGetAddresses(bool internal = false) { return false; } + + /** Upgrades the wallet to the specified version */ + virtual bool Upgrade(int prev_version, std::string& error) { return false; } + + virtual bool HavePrivateKeys() const { return false; } + + //! The action to do when the DB needs rewrite + virtual void RewriteDB() {} + + virtual int64_t GetOldestKeyPoolTime() { return GetTime(); } + + virtual size_t KeypoolCountExternalKeys() { return 0; } + virtual unsigned int GetKeyPoolSize() const { return 0; } + + virtual int64_t GetTimeFirstKey() const { return 0; } + + //! Return address metadata + virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } +}; + +class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProvider +{ +private: + using WatchOnlySet = std::set<CScript>; + using WatchKeyMap = std::map<CKeyID, CPubKey>; + + WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; + + using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; + + CryptedKeyMap mapCryptedKeys GUARDED_BY(cs_KeyStore); + WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); + WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); + + int64_t nTimeFirstKey GUARDED_BY(cs_wallet) = 0; + + bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); + bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + + /** + * Private version of AddWatchOnly method which does not accept a + * timestamp, and which will reset the wallet's nTimeFirstKey value to 1 if + * the watch key did not previously have a timestamp associated with it. + * Because this is an inherited virtual method, it is accessible despite + * being marked private, but it is marked private anyway to encourage use + * of the other AddWatchOnly which accepts a timestamp and sets + * nTimeFirstKey more intelligently for more efficient rescans. + */ + bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnlyInMem(const CScript &dest); + //! Adds a watch-only address to the store, and saves it to disk. + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Adds a key to the store, and saves it to disk. + bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); + + //! Adds a script to the store and saves it to disk + bool AddCScriptWithDB(WalletBatch& batch, const CScript& script); + + /** Add a KeyOriginInfo to the wallet */ + bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); + + /* the HD chain data model (external chain counters) */ + CHDChain hdChain; + + /* HD derive new child key (on internal or external chain) */ + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); + std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); + int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; + std::map<CKeyID, int64_t> m_pool_key_to_index; + + //! Fetches a key from the keypool + bool GetKeyFromPool(CPubKey &key, bool internal = false); + + /** + * Reserves a key from the keypool and sets nIndex to its index + * + * @param[out] nIndex the index of the key in keypool + * @param[out] keypool the keypool the key was drawn from, which could be the + * the pre-split pool if present, or the internal or external pool + * @param fRequestedInternal true if the caller would like the key drawn + * from the internal keypool, false if external is preferred + * + * @return true if succeeded, false if failed due to empty keypool + * @throws std::runtime_error if keypool read failed, key was invalid, + * was not found in the wallet, or was misclassified in the internal + * or external keypool + */ + bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); + + void KeepKey(int64_t nIndex); + void ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey); + +public: + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + isminetype IsMine(const CScript& script) const override; + + //! will encrypt previously unencrypted keys + bool EncryptKeys(CKeyingMaterial& vMasterKeyIn); + + bool GetReservedDestination(const OutputType type, bool internal, int64_t& index, CKeyPool& keypool) override; + void KeepDestination(int64_t index) override; + void ReturnDestination(int64_t index, bool internal, const CPubKey& pubkey) override; + + bool TopUp(unsigned int size = 0) override; + + void MarkUnusedAddresses(const CScript& script) override; + + //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo + void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + bool IsHDEnabled() const override; + + bool SetupGeneration(bool force = false) override; + + bool Upgrade(int prev_version, std::string& error) override; + + bool HavePrivateKeys() const override; + + void RewriteDB() override; + + int64_t GetOldestKeyPoolTime() override; + size_t KeypoolCountExternalKeys() override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + unsigned int GetKeyPoolSize() const override; + + int64_t GetTimeFirstKey() const override; + + const CKeyMetadata* GetMetadata(const CTxDestination& dest) const override; + + bool CanGetAddresses(bool internal = false) override; + + // Map from Key ID to key metadata. + std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_wallet); + + // Map from Script ID to key metadata (for watch-only keys). + std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); + + //! Adds a key to the store, and saves it to disk. + bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Adds a key to the store, without saving it to disk (used by LoadWallet) + bool LoadKey(const CKey& key, const CPubKey &pubkey); + //! Adds an encrypted key to the store, and saves it to disk. + bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) + bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); + void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Adds a CScript to the store + bool LoadCScript(const CScript& redeemScript); + //! Load metadata (used by LoadWallet) + void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Generate a new key + CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /* Set the HD chain model (chain child index counters) */ + void SetHDChain(const CHDChain& chain, bool memonly); + const CHDChain& GetHDChain() const { return hdChain; } + + //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) + bool LoadWatchOnly(const CScript &dest); + //! Returns whether the watch-only script is in the wallet + bool HaveWatchOnly(const CScript &dest) const; + //! Returns whether there are any watch-only things in the wallet + bool HaveWatchOnly() const; + //! Remove a watch only script from the keystore + bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + //! Fetches a pubkey from mapWatchKeys if it exists there + bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; + + /* SigningProvider overrides */ + bool HaveKey(const CKeyID &address) const override; + bool GetKey(const CKeyID &address, CKey& keyOut) const override; + bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; + bool AddCScript(const CScript& redeemScript) override; + bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + + //! Load a keypool entry + void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool NewKeyPool(); + void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /* Returns true if the wallet can generate new keys */ + bool CanGenerateKeys(); + + /* Generates a new HD seed (will not be activated) */ + CPubKey GenerateNewSeed(); + + /* Derives a new HD seed (will not be activated) */ + CPubKey DeriveNewSeed(const CKey& key); + + /* Set the current HD seed (will reset the chain child index counters) + Sets the seed's version based on the current wallet version (so the + caller must ensure the current wallet version is correct before calling + this function). */ + void SetHDSeed(const CPubKey& key); + + /** + * Explicitly make the wallet learn the related scripts for outputs to the + * given key. This is purely to make the wallet file compatible with older + * software, as FillableSigningProvider automatically does this implicitly for all + * keys now. + */ + void LearnRelatedScripts(const CPubKey& key, OutputType); + + /** + * Same as LearnRelatedScripts, but when the OutputType is not known (and could + * be anything). + */ + void LearnAllRelatedScripts(const CPubKey& key); + + /** + * Marks all keys in the keypool up to and including reserve_key as used. + */ + void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } + + std::set<CKeyID> GetKeys() const override; + // Temporary CWallet accessors and aliases. + friend class CWallet; + friend class ReserveDestination; + LegacyScriptPubKeyMan(CWallet& wallet); + bool SetCrypted(); + bool IsCrypted() const; + void NotifyWatchonlyChanged(bool fHaveWatchOnly) const; + void NotifyCanGetAddressesChanged() const; + template<typename... Params> void WalletLogPrintf(const std::string& fmt, const Params&... parameters) const; + CWallet& m_wallet; + CCriticalSection& cs_wallet; + CKeyingMaterial& vMasterKey GUARDED_BY(cs_KeyStore); + std::atomic<bool>& fUseCrypto; + bool& fDecryptionThoroughlyChecked; +}; + +#endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 9e7f0ed773..bc068f1499 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -2,13 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <node/context.h> #include <wallet/wallet.h> #include <wallet/coinselection.h> #include <wallet/coincontrol.h> #include <amount.h> #include <primitives/transaction.h> #include <random.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> @@ -28,7 +29,8 @@ std::vector<std::unique_ptr<CWalletTx>> wtxn; typedef std::set<CInputCoin> CoinSet; static std::vector<COutput> vCoins; -static auto testChain = interfaces::MakeChain(); +static NodeContext testNode; +static auto testChain = interfaces::MakeChain(testNode); static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); static CAmount balance = 0; diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index c961456572..f4a4c9fa7c 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -7,7 +7,7 @@ #include <boost/test/unit_test.hpp> #include <fs.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/db.h> diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index e2b7075085..6ba7d66b7c 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -6,7 +6,8 @@ #define BITCOIN_WALLET_TEST_INIT_TEST_FIXTURE_H #include <interfaces/chain.h> -#include <test/setup_common.h> +#include <node/context.h> +#include <test/util/setup_common.h> struct InitWalletDirTestingSetup: public BasicTestingSetup { @@ -17,7 +18,8 @@ struct InitWalletDirTestingSetup: public BasicTestingSetup { fs::path m_datadir; fs::path m_cwd; std::map<std::string, fs::path> m_walletdir_path_cases; - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + NodeContext m_node; + std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::ChainClient> m_chain_client; }; diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 279542ffad..c228e06009 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -5,7 +5,8 @@ #include <boost/test/unit_test.hpp> #include <noui.h> -#include <test/setup_common.h> +#include <test/util/logging.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <wallet/test/init_test_fixture.h> @@ -34,28 +35,31 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) { SetWalletDir(m_walletdir_path_cases["nonexistent"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("does not exist"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_directory) { SetWalletDir(m_walletdir_path_cases["file"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is not a directory"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative) { SetWalletDir(m_walletdir_path_cases["relative"]); - noui_suppress(); - bool result = m_chain_client->verify(); - noui_reconnect(); - BOOST_CHECK(result == false); + { + ASSERT_DEBUG_LOG("is a relative path"); + bool result = m_chain_client->verify(); + BOOST_CHECK(result == false); + } } BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 062fef7748..76c3639d16 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -3,9 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> +#include <node/context.h> #include <script/script.h> #include <script/standard.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/ismine.h> #include <wallet/wallet.h> @@ -26,7 +27,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CKey uncompressedKey; uncompressedKey.MakeNewKey(false); CPubKey uncompressedPubkey = uncompressedKey.GetPubKey(); - std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(); + NodeContext node; + std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); CScript scriptPubKey; isminetype result; @@ -38,12 +40,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -54,12 +56,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -70,12 +72,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -86,12 +88,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); // Keystore does not have key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -104,17 +106,17 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); // Keystore does not have redeemScript or key - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has redeemScript but no key - BOOST_CHECK(keystore.AddCScript(redeemScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has redeemScript and key - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -127,11 +129,11 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CScript redeemscript = GetScriptForDestination(ScriptHash(redeemscript_inner)); scriptPubKey = GetScriptForDestination(ScriptHash(redeemscript)); - BOOST_CHECK(keystore.AddCScript(redeemscript)); - BOOST_CHECK(keystore.AddCScript(redeemscript_inner)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemscript_inner)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -144,11 +146,11 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CScript witnessscript = GetScriptForDestination(ScriptHash(redeemscript)); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); - BOOST_CHECK(keystore.AddCScript(witnessscript)); - BOOST_CHECK(keystore.AddCScript(redeemscript)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -160,10 +162,10 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CScript witnessscript = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); - BOOST_CHECK(keystore.AddCScript(witnessscript)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -176,11 +178,11 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CScript witnessscript = GetScriptForDestination(WitnessV0ScriptHash(witnessscript_inner)); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); - BOOST_CHECK(keystore.AddCScript(witnessscript_inner)); - BOOST_CHECK(keystore.AddCScript(witnessscript)); - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - BOOST_CHECK(keystore.AddKey(keys[0])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript_inner)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessscript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -188,13 +190,13 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); // Keystore implicitly has key and P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -202,17 +204,17 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(uncompressedPubkey))); // Keystore has key, but no P2SH redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has key and P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -224,25 +226,25 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); // Keystore does not have any keys - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has 1/2 keys - BOOST_CHECK(keystore.AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has 2/2 keys - BOOST_CHECK(keystore.AddKey(keys[1])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has 2/2 keys and the script - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -250,19 +252,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - BOOST_CHECK(keystore.AddKey(keys[1])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); CScript redeemScript = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); // Keystore has no redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has redeemScript - BOOST_CHECK(keystore.AddCScript(redeemScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -270,24 +272,24 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); - BOOST_CHECK(keystore.AddKey(keys[1])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); CScript witnessScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); // Keystore has keys, but no witnessScript or P2SH redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys and witnessScript, but no P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(witnessScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys, witnessScript, P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -295,24 +297,24 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(uncompressedKey)); - BOOST_CHECK(keystore.AddKey(keys[1])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); CScript witnessScript = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); // Keystore has keys, but no witnessScript or P2SH redeemScript - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys and witnessScript, but no P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(witnessScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys, witnessScript, P2SH redeemScript - BOOST_CHECK(keystore.AddCScript(scriptPubKey)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(scriptPubKey)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -326,19 +328,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); // Keystore has no witnessScript, P2SH redeemScript, or keys - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has witnessScript and P2SH redeemScript, but no keys - BOOST_CHECK(keystore.AddCScript(redeemScript)); - BOOST_CHECK(keystore.AddCScript(witnessScript)); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(redeemScript)); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddCScript(witnessScript)); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); // Keystore has keys, witnessScript, P2SH redeemScript - BOOST_CHECK(keystore.AddKey(keys[0])); - BOOST_CHECK(keystore.AddKey(keys[1])); - result = IsMine(keystore, scriptPubKey); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); } @@ -346,12 +348,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_RETURN << ToByteVector(pubkeys[0]); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -359,12 +361,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_0 << ToByteVector(ParseHex("aabb")); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -372,12 +374,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_16 << ToByteVector(ParseHex("aabb")); - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } @@ -385,12 +387,12 @@ BOOST_AUTO_TEST_CASE(ismine_standard) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); LOCK(keystore.cs_wallet); - BOOST_CHECK(keystore.AddKey(keys[0])); + BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); scriptPubKey << OP_9 << OP_ADD << OP_11 << OP_EQUAL; - result = IsMine(keystore, scriptPubKey); + result = keystore.GetLegacyScriptPubKeyMan()->IsMine(scriptPubKey); BOOST_CHECK_EQUAL(result, ISMINE_NO); } } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 0400f1207c..d930ca6bea 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -9,13 +9,14 @@ #include <wallet/wallet.h> #include <boost/test/unit_test.hpp> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <wallet/test/wallet_test_fixture.h> BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) BOOST_AUTO_TEST_CASE(psbt_updater_test) { + auto spk_man = m_wallet.GetLegacyScriptPubKeyMan(); LOCK(m_wallet.cs_wallet); // Create prevtxs and add to wallet @@ -35,23 +36,23 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) CScript rs1; CDataStream s_rs1(ParseHex("475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae"), SER_NETWORK, PROTOCOL_VERSION); s_rs1 >> rs1; - m_wallet.AddCScript(rs1); + spk_man->AddCScript(rs1); CScript rs2; CDataStream s_rs2(ParseHex("2200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903"), SER_NETWORK, PROTOCOL_VERSION); s_rs2 >> rs2; - m_wallet.AddCScript(rs2); + spk_man->AddCScript(rs2); CScript ws1; CDataStream s_ws1(ParseHex("47522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae"), SER_NETWORK, PROTOCOL_VERSION); s_ws1 >> ws1; - m_wallet.AddCScript(ws1); + spk_man->AddCScript(ws1); // Add hd seed CKey key = DecodeSecret("5KSSJQ7UNfFGwVgpCZDSHm5rVNhMFcFtvWM3zQ8mW4qNDEN7LFd"); // Mainnet and uncompressed form of cUkG8i1RFfWGWy5ziR11zJ5V4U4W3viSFCfyJmZnvQaUsd1xuF3T - CPubKey master_pub_key = m_wallet.DeriveNewSeed(key); - m_wallet.SetHDSeed(master_pub_key); - m_wallet.NewKeyPool(); + CPubKey master_pub_key = spk_man->DeriveNewSeed(key); + spk_man->SetHDSeed(master_pub_key); + spk_man->NewKeyPool(); // Call FillPSBT PartiallySignedTransaction psbtx; diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 2f41813234..97f8c94fa6 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <wallet/crypter.h> diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index c1dbecdf8c..4e4129fb2c 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H #define BITCOIN_WALLET_TEST_WALLET_TEST_FIXTURE_H -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <interfaces/chain.h> #include <interfaces/wallet.h> +#include <node/context.h> #include <wallet/wallet.h> #include <memory> @@ -18,7 +19,8 @@ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + NodeContext m_node; + std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, {}); CWallet m_wallet; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 73523ca36d..2f21b2439b 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -8,11 +8,11 @@ #include <stdint.h> #include <vector> -#include <consensus/validation.h> #include <interfaces/chain.h> +#include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> -#include <test/setup_common.h> +#include <test/util/setup_common.h> #include <validation.h> #include <wallet/coincontrol.h> #include <wallet/test/wallet_test_fixture.h> @@ -28,8 +28,10 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static void AddKey(CWallet& wallet, const CKey& key) { + auto spk_man = wallet.GetLegacyScriptPubKeyMan(); LOCK(wallet.cs_wallet); - wallet.AddKeyPubKey(key, key.GetPubKey()); + AssertLockHeld(spk_man->cs_wallet); + spk_man->AddKeyPubKey(key, key.GetPubKey()); } BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) @@ -40,13 +42,18 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = ::ChainActive().Tip(); - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); // Verify ScanForWalletTransactions accommodates a null start block. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -62,6 +69,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // and new block files. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -81,6 +92,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // file. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -99,6 +114,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + { + LOCK(wallet.cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(&wallet); reserver.reserve(); @@ -119,7 +138,8 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = ::ChainActive().Tip(); - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); @@ -186,7 +206,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); @@ -195,9 +216,11 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + auto spk_man = wallet->GetLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); - wallet->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; - wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + AssertLockHeld(spk_man->cs_wallet); + spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); JSONRPCRequest request; request.params.setArray(); @@ -240,26 +263,31 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + auto spk_man = wallet.GetLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); LOCK(wallet.cs_wallet); + AssertLockHeld(spk_man->cs_wallet); + wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); - wtx.SetConf(CWalletTx::Status::CONFIRMED, ::ChainActive().Tip()->GetBlockHash(), 0); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 0); + wtx.m_confirm = confirm; // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(*locked_chain), 0); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); - // Invalidate the cached value, add the key, and make sure a new immature + // Invalidate the cached vanue, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); - wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(*locked_chain), 50*COIN); + BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); + BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); } static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) @@ -290,7 +318,8 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 wallet.AddToWallet(wtx); } if (block) { - wtx.SetConf(CWalletTx::Status::CONFIRMED, block->GetBlockHash(), 0); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block->nHeight, block->GetBlockHash(), 0); + wtx.m_confirm = confirm; } wallet.AddToWallet(wtx); return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; @@ -328,9 +357,10 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests) { CTxDestination dest = PKHash(); LOCK(m_wallet.cs_wallet); - m_wallet.AddDestData(dest, "misc", "val_misc"); - m_wallet.AddDestData(dest, "rr0", "val_rr0"); - m_wallet.AddDestData(dest, "rr1", "val_rr1"); + WalletBatch batch{m_wallet.GetDatabase()}; + m_wallet.AddDestData(batch, dest, "misc", "val_misc"); + m_wallet.AddDestData(batch, dest, "rr0", "val_rr0"); + m_wallet.AddDestData(batch, dest, "rr1", "val_rr1"); auto values = m_wallet.GetDestValues("rr"); BOOST_CHECK_EQUAL(values.size(), 2U); @@ -338,37 +368,38 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests) BOOST_CHECK_EQUAL(values[1], "val_rr1"); } -// Test some watch-only wallet methods by the procedure of loading (LoadWatchOnly), +// Test some watch-only LegacyScriptPubKeyMan methods by the procedure of loading (LoadWatchOnly), // checking (HaveWatchOnly), getting (GetWatchPubKey) and removing (RemoveWatchOnly) a // given PubKey, resp. its corresponding P2PK Script. Results of the the impact on // the address -> PubKey map is dependent on whether the PubKey is a point on the curve -static void TestWatchOnlyPubKey(CWallet& wallet, const CPubKey& add_pubkey) +static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& add_pubkey) { CScript p2pk = GetScriptForRawPubKey(add_pubkey); CKeyID add_address = add_pubkey.GetID(); CPubKey found_pubkey; - LOCK(wallet.cs_wallet); + LOCK(spk_man->cs_wallet); // all Scripts (i.e. also all PubKeys) are added to the general watch-only set - BOOST_CHECK(!wallet.HaveWatchOnly(p2pk)); - wallet.LoadWatchOnly(p2pk); - BOOST_CHECK(wallet.HaveWatchOnly(p2pk)); + BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); + spk_man->LoadWatchOnly(p2pk); + BOOST_CHECK(spk_man->HaveWatchOnly(p2pk)); // only PubKeys on the curve shall be added to the watch-only address -> PubKey map bool is_pubkey_fully_valid = add_pubkey.IsFullyValid(); if (is_pubkey_fully_valid) { - BOOST_CHECK(wallet.GetWatchPubKey(add_address, found_pubkey)); + BOOST_CHECK(spk_man->GetWatchPubKey(add_address, found_pubkey)); BOOST_CHECK(found_pubkey == add_pubkey); } else { - BOOST_CHECK(!wallet.GetWatchPubKey(add_address, found_pubkey)); + BOOST_CHECK(!spk_man->GetWatchPubKey(add_address, found_pubkey)); BOOST_CHECK(found_pubkey == CPubKey()); // passed key is unchanged } - wallet.RemoveWatchOnly(p2pk); - BOOST_CHECK(!wallet.HaveWatchOnly(p2pk)); + AssertLockHeld(spk_man->cs_wallet); + spk_man->RemoveWatchOnly(p2pk); + BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); if (is_pubkey_fully_valid) { - BOOST_CHECK(!wallet.GetWatchPubKey(add_address, found_pubkey)); + BOOST_CHECK(!spk_man->GetWatchPubKey(add_address, found_pubkey)); BOOST_CHECK(found_pubkey == add_pubkey); // passed key is unchanged } } @@ -383,37 +414,38 @@ static void PollutePubKey(CPubKey& pubkey) assert(pubkey.IsValid()); } -// Test watch-only wallet logic for PubKeys +// Test watch-only logic for PubKeys BOOST_AUTO_TEST_CASE(WatchOnlyPubKeys) { CKey key; CPubKey pubkey; + LegacyScriptPubKeyMan* spk_man = m_wallet.GetLegacyScriptPubKeyMan(); - BOOST_CHECK(!m_wallet.HaveWatchOnly()); + BOOST_CHECK(!spk_man->HaveWatchOnly()); // uncompressed valid PubKey key.MakeNewKey(false); pubkey = key.GetPubKey(); assert(!pubkey.IsCompressed()); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // uncompressed cryptographically invalid PubKey PollutePubKey(pubkey); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // compressed valid PubKey key.MakeNewKey(true); pubkey = key.GetPubKey(); assert(pubkey.IsCompressed()); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // compressed cryptographically invalid PubKey PollutePubKey(pubkey); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); // invalid empty PubKey pubkey = CPubKey(); - TestWatchOnlyPubKey(m_wallet, pubkey); + TestWatchOnlyPubKey(spk_man, pubkey); } class ListCoinsTestingSetup : public TestChain100Setup @@ -423,6 +455,10 @@ public: { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + { + LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); @@ -451,8 +487,7 @@ public: auto locked_chain = m_chain->lock(); BOOST_CHECK(wallet->CreateTransaction(*locked_chain, {recipient}, tx, fee, changePos, error, dummy)); } - CValidationState state; - BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, state)); + wallet->CommitTransaction(tx, {}, {}); CMutableTransaction blocktx; { LOCK(wallet->cs_wallet); @@ -462,13 +497,16 @@ public: LOCK(cs_main); LOCK(wallet->cs_wallet); + wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, ::ChainActive().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - it->second.SetConf(CWalletTx::Status::CONFIRMED, ::ChainActive().Tip()->GetBlockHash(), 1); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 1); + it->second.m_confirm = confirm; return it->second; } - std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(); + NodeContext m_node; + std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); std::unique_ptr<CWallet> wallet; }; @@ -540,7 +578,8 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { - auto chain = interfaces::MakeChain(); + NodeContext node; + auto chain = interfaces::MakeChain(node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6adcf15167..2f2931cef1 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -210,9 +210,14 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& } // Set a seed for the wallet - CPubKey master_pub_key = wallet->GenerateNewSeed(); - wallet->SetHDSeed(master_pub_key); - wallet->NewKeyPool(); + { + if (auto spk_man = wallet->m_spk_man.get()) { + if (!spk_man->SetupGeneration()) { + error = "Unable to generate initial keys"; + return WalletCreationStatus::CREATION_FAILED; + } + } + } // Relock the wallet wallet->Lock(); @@ -224,8 +229,6 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& return WalletCreationStatus::SUCCESS; } -const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; - const uint256 CWalletTx::ABANDON_HASH(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); /** @defgroup mapWallet @@ -238,18 +241,6 @@ std::string COutput::ToString() const return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider) -{ - std::vector<CScript> dummy; - FlatSigningProvider out; - InferDescriptor(spk, provider)->Expand(0, DUMMY_SIGNING_PROVIDER, dummy, out); - std::vector<CKeyID> ret; - for (const auto& entry : out.pubkeys) { - ret.push_back(entry.first); - } - return ret; -} - const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { LOCK(cs_wallet); @@ -259,356 +250,19 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const return &(it->second); } -CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) -{ - assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - assert(!IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); - AssertLockHeld(cs_wallet); - bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets - - CKey secret; - - // Create new metadata - int64_t nCreationTime = GetTime(); - CKeyMetadata metadata(nCreationTime); - - // use HD key derivation if HD was enabled during wallet creation and a seed is present - if (IsHDEnabled()) { - DeriveNewChildKey(batch, metadata, secret, (CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); - } else { - secret.MakeNewKey(fCompressed); - } - - // Compressed public keys were introduced in version 0.6.0 - if (fCompressed) { - SetMinVersion(FEATURE_COMPRPUBKEY); - } - - CPubKey pubkey = secret.GetPubKey(); - assert(secret.VerifyPubKey(pubkey)); - - mapKeyMetadata[pubkey.GetID()] = metadata; - UpdateTimeFirstKey(nCreationTime); - - if (!AddKeyPubKeyWithDB(batch, secret, pubkey)) { - throw std::runtime_error(std::string(__func__) + ": AddKey failed"); - } - return pubkey; -} - -void CWallet::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal) -{ - // for now we use a fixed keypath scheme of m/0'/0'/k - CKey seed; //seed (256bit) - CExtKey masterKey; //hd master key - CExtKey accountKey; //key at m/0' - CExtKey chainChildKey; //key at m/0'/0' (external) or m/0'/1' (internal) - CExtKey childKey; //key at m/0'/0'/<n>' - - // try to get the seed - if (!GetKey(hdChain.seed_id, seed)) - throw std::runtime_error(std::string(__func__) + ": seed not found"); - - masterKey.SetSeed(seed.begin(), seed.size()); - - // derive m/0' - // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) - masterKey.Derive(accountKey, BIP32_HARDENED_KEY_LIMIT); - - // derive m/0'/0' (external chain) OR m/0'/1' (internal chain) - assert(internal ? CanSupportFeature(FEATURE_HD_SPLIT) : true); - accountKey.Derive(chainChildKey, BIP32_HARDENED_KEY_LIMIT+(internal ? 1 : 0)); - - // derive child key at next index, skip keys already known to the wallet - do { - // always derive hardened keys - // childIndex | BIP32_HARDENED_KEY_LIMIT = derive childIndex in hardened child-index-range - // example: 1 | BIP32_HARDENED_KEY_LIMIT == 0x80000001 == 2147483649 - if (internal) { - chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; - metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nInternalChainCounter++; - } - else { - chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; - metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); - metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); - hdChain.nExternalChainCounter++; - } - } while (HaveKey(childKey.key.GetPubKey().GetID())); - secret = childKey.key; - metadata.hd_seed_id = hdChain.seed_id; - CKeyID master_id = masterKey.key.GetPubKey().GetID(); - std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); - metadata.has_key_origin = true; - // update the chain model in the database - if (!batch.WriteHDChain(hdChain)) - throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); -} - -bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) -{ - AssertLockHeld(cs_wallet); - - // Make sure we aren't adding private keys to private key disabled wallets - assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - - // FillableSigningProvider has no concept of wallet databases, but calls AddCryptedKey - // which is overridden below. To avoid flushes, the database handle is - // tunneled through to it. - bool needsDB = !encrypted_batch; - if (needsDB) { - encrypted_batch = &batch; - } - if (!AddKeyPubKeyInner(secret, pubkey)) { - if (needsDB) encrypted_batch = nullptr; - return false; - } - if (needsDB) encrypted_batch = nullptr; - - // check if we need to remove from watch-only - CScript script; - script = GetScriptForDestination(PKHash(pubkey)); - if (HaveWatchOnly(script)) { - RemoveWatchOnly(script); - } - script = GetScriptForRawPubKey(pubkey); - if (HaveWatchOnly(script)) { - RemoveWatchOnly(script); - } - - if (!IsCrypted()) { - return batch.WriteKey(pubkey, - secret.GetPrivKey(), - mapKeyMetadata[pubkey.GetID()]); - } - UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); - return true; -} - -bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) -{ - WalletBatch batch(*database); - return CWallet::AddKeyPubKeyWithDB(batch, secret, pubkey); -} - -bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, - const std::vector<unsigned char> &vchCryptedSecret) -{ - if (!AddCryptedKeyInner(vchPubKey, vchCryptedSecret)) - return false; - { - LOCK(cs_wallet); - if (encrypted_batch) - return encrypted_batch->WriteCryptedKey(vchPubKey, - vchCryptedSecret, - mapKeyMetadata[vchPubKey.GetID()]); - else - return WalletBatch(*database).WriteCryptedKey(vchPubKey, - vchCryptedSecret, - mapKeyMetadata[vchPubKey.GetID()]); - } -} - -void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) -{ - AssertLockHeld(cs_wallet); - UpdateTimeFirstKey(meta.nCreateTime); - mapKeyMetadata[keyID] = meta; -} - -void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) -{ - AssertLockHeld(cs_wallet); - UpdateTimeFirstKey(meta.nCreateTime); - m_script_metadata[script_id] = meta; -} - void CWallet::UpgradeKeyMetadata() { - AssertLockHeld(cs_wallet); if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { return; } - std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database); - for (auto& meta_pair : mapKeyMetadata) { - CKeyMetadata& meta = meta_pair.second; - if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin - CKey key; - GetKey(meta.hd_seed_id, key); - CExtKey masterKey; - masterKey.SetSeed(key.begin(), key.size()); - // Add to map - CKeyID master_id = masterKey.key.GetPubKey().GetID(); - std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); - if (!ParseHDKeypath(meta.hdKeypath, meta.key_origin.path)) { - throw std::runtime_error("Invalid stored hdKeypath"); - } - meta.has_key_origin = true; - if (meta.nVersion < CKeyMetadata::VERSION_WITH_KEY_ORIGIN) { - meta.nVersion = CKeyMetadata::VERSION_WITH_KEY_ORIGIN; - } - - // Write meta to wallet - CPubKey pubkey; - if (GetPubKey(meta_pair.first, pubkey)) { - batch->WriteKeyMetadata(meta, pubkey, true); - } - } + if (m_spk_man) { + AssertLockHeld(m_spk_man->cs_wallet); + m_spk_man->UpgradeKeyMetadata(); } - batch.reset(); //write before setting the flag SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } -bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) -{ - return AddCryptedKeyInner(vchPubKey, vchCryptedSecret); -} - -/** - * Update wallet first key creation time. This should be called whenever keys - * are added to the wallet, with the oldest key creation time. - */ -void CWallet::UpdateTimeFirstKey(int64_t nCreateTime) -{ - AssertLockHeld(cs_wallet); - if (nCreateTime <= 1) { - // Cannot determine birthday information, so set the wallet birthday to - // the beginning of time. - nTimeFirstKey = 1; - } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { - nTimeFirstKey = nCreateTime; - } -} - -bool CWallet::AddCScript(const CScript& redeemScript) -{ - WalletBatch batch(*database); - return AddCScriptWithDB(batch, redeemScript); -} - -bool CWallet::AddCScriptWithDB(WalletBatch& batch, const CScript& redeemScript) -{ - if (!FillableSigningProvider::AddCScript(redeemScript)) - return false; - if (batch.WriteCScript(Hash160(redeemScript), redeemScript)) { - UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); - return true; - } - return false; -} - -bool CWallet::LoadCScript(const CScript& redeemScript) -{ - /* A sanity check was added in pull #3843 to avoid adding redeemScripts - * that never can be redeemed. However, old wallets may still contain - * these. Do not add them to the wallet and warn. */ - if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) - { - std::string strAddr = EncodeDestination(ScriptHash(redeemScript)); - WalletLogPrintf("%s: Warning: This wallet contains a redeemScript of size %i which exceeds maximum size %i thus can never be redeemed. Do not use address %s.\n", __func__, redeemScript.size(), MAX_SCRIPT_ELEMENT_SIZE, strAddr); - return true; - } - - return FillableSigningProvider::AddCScript(redeemScript); -} - -static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) -{ - std::vector<std::vector<unsigned char>> solutions; - return Solver(dest, solutions) == TX_PUBKEY && - (pubKeyOut = CPubKey(solutions[0])).IsFullyValid(); -} - -bool CWallet::AddWatchOnlyInMem(const CScript &dest) -{ - LOCK(cs_KeyStore); - setWatchOnly.insert(dest); - CPubKey pubKey; - if (ExtractPubKey(dest, pubKey)) { - mapWatchKeys[pubKey.GetID()] = pubKey; - ImplicitlyLearnRelatedKeyScripts(pubKey); - } - return true; -} - -bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) -{ - if (!AddWatchOnlyInMem(dest)) - return false; - const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; - UpdateTimeFirstKey(meta.nCreateTime); - NotifyWatchonlyChanged(true); - if (batch.WriteWatchOnly(dest, meta)) { - UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); - return true; - } - return false; -} - -bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) -{ - m_script_metadata[CScriptID(dest)].nCreateTime = create_time; - return AddWatchOnlyWithDB(batch, dest); -} - -bool CWallet::AddWatchOnly(const CScript& dest) -{ - WalletBatch batch(*database); - return AddWatchOnlyWithDB(batch, dest); -} - -bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) -{ - m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; - return AddWatchOnly(dest); -} - -bool CWallet::RemoveWatchOnly(const CScript &dest) -{ - AssertLockHeld(cs_wallet); - { - LOCK(cs_KeyStore); - setWatchOnly.erase(dest); - CPubKey pubKey; - if (ExtractPubKey(dest, pubKey)) { - mapWatchKeys.erase(pubKey.GetID()); - } - // Related CScripts are not removed; having superfluous scripts around is - // harmless (see comment in ImplicitlyLearnRelatedKeyScripts). - } - - if (!HaveWatchOnly()) - NotifyWatchonlyChanged(false); - if (!WalletBatch(*database).EraseWatchOnly(dest)) - return false; - - return true; -} - -bool CWallet::LoadWatchOnly(const CScript &dest) -{ - return AddWatchOnlyInMem(dest); -} - -bool CWallet::HaveWatchOnly(const CScript &dest) const -{ - LOCK(cs_KeyStore); - return setWatchOnly.count(dest) > 0; -} - -bool CWallet::HaveWatchOnly() const -{ - LOCK(cs_KeyStore); - return (!setWatchOnly.empty()); -} - bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_keys) { CCrypter crypter; @@ -798,7 +452,7 @@ void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> ran * Outpoint is spent if any non-conflicted transaction * spends it: */ -bool CWallet::IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash, unsigned int n) const +bool CWallet::IsSpent(const uint256& hash, unsigned int n) const { const COutPoint outpoint(hash, n); std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range; @@ -809,7 +463,7 @@ bool CWallet::IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash const uint256& wtxid = it->second; std::map<uint256, CWalletTx>::const_iterator mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { - int depth = mit->second.GetDepthInMainChain(locked_chain); + int depth = mit->second.GetDepthInMainChain(); if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) return true; // Spent } @@ -887,14 +541,15 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) } encrypted_batch->WriteMasterKey(nMasterKeyMaxID, kMasterKey); - if (!EncryptKeys(_vMasterKey)) - { - encrypted_batch->TxnAbort(); - delete encrypted_batch; - encrypted_batch = nullptr; - // We now probably have half of our keys encrypted in memory, and half not... - // die and let the user reload the unencrypted wallet. - assert(false); + if (auto spk_man = m_spk_man.get()) { + if (!spk_man->EncryptKeys(_vMasterKey)) { + encrypted_batch->TxnAbort(); + delete encrypted_batch; + encrypted_batch = nullptr; + // We now probably have half of our keys encrypted in memory, and half not... + // die and let the user reload the unencrypted wallet. + assert(false); + } } // Encryption was introduced in version 0.4.0 @@ -915,11 +570,13 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Unlock(strWalletPassphrase); // if we are using HD, replace the HD seed with a new one - if (IsHDEnabled()) { - SetHDSeed(GenerateNewSeed()); + if (auto spk_man = m_spk_man.get()) { + if (spk_man->IsHDEnabled()) { + if (!spk_man->SetupGeneration(true)) { + return false; + } + } } - - NewKeyPool(); Lock(); // Need to completely rewrite the wallet file; if we don't, bdb might keep @@ -1044,19 +701,19 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) return success; } -void CWallet::SetUsedDestinationState(const uint256& hash, unsigned int n, bool used) +void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used) { + AssertLockHeld(cs_wallet); const CWalletTx* srctx = GetWalletTx(hash); if (!srctx) return; CTxDestination dst; if (ExtractDestination(srctx->tx->vout[n].scriptPubKey, dst)) { - if (::IsMine(*this, dst)) { - LOCK(cs_wallet); + if (IsMine(dst)) { if (used && !GetDestData(dst, "used", nullptr)) { - AddDestData(dst, "used", "p"); // p for "present", opposite of absent (null) + AddDestData(batch, dst, "used", "p"); // p for "present", opposite of absent (null) } else if (!used && GetDestData(dst, "used", nullptr)) { - EraseDestData(dst, "used"); + EraseDestData(batch, dst, "used"); } } } @@ -1065,7 +722,7 @@ void CWallet::SetUsedDestinationState(const uint256& hash, unsigned int n, bool bool CWallet::IsUsedDestination(const CTxDestination& dst) const { LOCK(cs_wallet); - return ::IsMine(*this, dst) && GetDestData(dst, "used", nullptr); + return IsMine(dst) && GetDestData(dst, "used", nullptr); } bool CWallet::IsUsedDestination(const uint256& hash, unsigned int n) const @@ -1087,7 +744,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) // Mark used destinations for (const CTxIn& txin : wtxIn.tx->vin) { const COutPoint& op = txin.prevout; - SetUsedDestinationState(op.hash, op.n, true); + SetUsedDestinationState(batch, op.hash, op.n, true); } } @@ -1111,10 +768,12 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) wtx.m_confirm.status = wtxIn.m_confirm.status; wtx.m_confirm.nIndex = wtxIn.m_confirm.nIndex; wtx.m_confirm.hashBlock = wtxIn.m_confirm.hashBlock; + wtx.m_confirm.block_height = wtxIn.m_confirm.block_height; fUpdated = true; } else { assert(wtx.m_confirm.nIndex == wtxIn.m_confirm.nIndex); assert(wtx.m_confirm.hashBlock == wtxIn.m_confirm.hashBlock); + assert(wtx.m_confirm.block_height == wtxIn.m_confirm.block_height); } if (wtxIn.fFromMe && wtxIn.fFromMe != wtx.fFromMe) { @@ -1165,12 +824,22 @@ void CWallet::LoadToWallet(CWalletTx& wtxIn) { // If wallet doesn't have a chain (e.g wallet-tool), lock can't be taken. auto locked_chain = LockChain(); - // If tx hasn't been reorged out of chain while wallet being shutdown - // change tx status to UNCONFIRMED and reset hashBlock/nIndex. - if (!wtxIn.m_confirm.hashBlock.IsNull()) { - if (locked_chain && !locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock)) { + if (locked_chain) { + Optional<int> block_height = locked_chain->getBlockHeight(wtxIn.m_confirm.hashBlock); + if (block_height) { + // Update cached block height variable since it not stored in the + // serialized transaction. + wtxIn.m_confirm.block_height = *block_height; + } else if (wtxIn.isConflicted() || wtxIn.isConfirmed()) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. wtxIn.setUnconfirmed(); wtxIn.m_confirm.hashBlock = uint256(); + wtxIn.m_confirm.block_height = 0; wtxIn.m_confirm.nIndex = 0; } } @@ -1187,25 +856,25 @@ void CWallet::LoadToWallet(CWalletTx& wtxIn) if (it != mapWallet.end()) { CWalletTx& prevtx = it->second; if (prevtx.isConflicted()) { - MarkConflicted(prevtx.m_confirm.hashBlock, wtx.GetHash()); + MarkConflicted(prevtx.m_confirm.hashBlock, prevtx.m_confirm.block_height, wtx.GetHash()); } } } } -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate) { const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (!block_hash.IsNull()) { + if (!confirm.hashBlock.IsNull()) { for (const CTxIn& txin : tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { if (range.first->second != tx.GetHash()) { - WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), block_hash.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); - MarkConflicted(block_hash, range.first->second); + WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), confirm.hashBlock.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); + MarkConflicted(confirm.hashBlock, confirm.block_height, range.first->second); } range.first++; } @@ -1224,17 +893,8 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::St // loop though all outputs for (const CTxOut& txout: tx.vout) { - // extract addresses and check if they match with an unused keypool key - for (const auto& keyid : GetAffectedKeys(txout.scriptPubKey, *this)) { - std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); - if (mi != m_pool_key_to_index.end()) { - WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); - MarkReserveKeysAsUsed(mi->second); - - if (!TopUpKeyPool()) { - WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); - } - } + if (auto spk_man = m_spk_man.get()) { + spk_man->MarkUnusedAddresses(txout.scriptPubKey); } } @@ -1242,7 +902,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::St // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - wtx.SetConf(status, block_hash, posInBlock); + wtx.m_confirm = confirm; return AddToWallet(wtx, false); } @@ -1255,7 +915,7 @@ bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const auto locked_chain = chain().lock(); LOCK(cs_wallet); const CWalletTx* wtx = GetWalletTx(hashTx); - return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain(*locked_chain) == 0 && !wtx->InMempool(); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() == 0 && !wtx->InMempool(); } void CWallet::MarkInputsDirty(const CTransactionRef& tx) @@ -1268,9 +928,9 @@ void CWallet::MarkInputsDirty(const CTransactionRef& tx) } } -bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx) +bool CWallet::AbandonTransaction(const uint256& hashTx) { - auto locked_chain_recursive = chain().lock(); // Temporary. Removed in upcoming lock cleanup + auto locked_chain = chain().lock(); // Temporary. Removed in upcoming lock cleanup LOCK(cs_wallet); WalletBatch batch(*database, "r+"); @@ -1282,7 +942,7 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); CWalletTx& origtx = it->second; - if (origtx.GetDepthInMainChain(locked_chain) != 0 || origtx.InMempool()) { + if (origtx.GetDepthInMainChain() != 0 || origtx.InMempool()) { return false; } @@ -1295,14 +955,13 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(locked_chain); + int currentconfirm = wtx.GetDepthInMainChain(); // If the orig tx was not in block, none of its spends can be assert(currentconfirm <= 0); // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} if (currentconfirm == 0 && !wtx.isAbandoned()) { // If the orig tx was not in block/mempool, none of its spends can be in mempool assert(!wtx.InMempool()); - wtx.m_confirm.nIndex = 0; wtx.setAbandoned(); wtx.MarkDirty(); batch.WriteTx(wtx); @@ -1324,12 +983,12 @@ bool CWallet::AbandonTransaction(interfaces::Chain::Lock& locked_chain, const ui return true; } -void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) +void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - int conflictconfirms = -locked_chain->getBlockDepth(hashBlock); + int conflictconfirms = (m_last_block_processed_height - conflicting_height + 1) * -1; // If number of conflict confirms cannot be determined, this means // that the block is still unknown or not yet part of the main chain, // for example when loading the wallet during a reindex. Do nothing in that @@ -1352,12 +1011,13 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(*locked_chain); + int currentconfirm = wtx.GetDepthInMainChain(); if (conflictconfirms < currentconfirm) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. wtx.m_confirm.nIndex = 0; wtx.m_confirm.hashBlock = hashBlock; + wtx.m_confirm.block_height = conflicting_height; wtx.setConflicted(); wtx.MarkDirty(); batch.WriteTx(wtx); @@ -1376,9 +1036,9 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool update_tx) +void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx) { - if (!AddToWalletIfInvolvingMe(ptx, status, block_hash, posInBlock, update_tx)) + if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1390,7 +1050,8 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Status stat void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - SyncTransaction(ptx, CWalletTx::Status::UNCONFIRMED, {} /* block hash */, 0 /* position in block */); + CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); + SyncTransaction(ptx, confirm); auto it = mapWallet.find(ptx->GetHash()); if (it != mapWallet.end()) { @@ -1406,23 +1067,26 @@ void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { } } -void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) { +void CWallet::BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted, int height) +{ const uint256& block_hash = block.GetHash(); auto locked_chain = chain().lock(); LOCK(cs_wallet); - for (size_t i = 0; i < block.vtx.size(); i++) { - SyncTransaction(block.vtx[i], CWalletTx::Status::CONFIRMED, block_hash, i); - TransactionRemovedFromMempool(block.vtx[i]); + m_last_block_processed_height = height; + m_last_block_processed = block_hash; + for (size_t index = 0; index < block.vtx.size(); index++) { + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, block_hash, index); + SyncTransaction(block.vtx[index], confirm); + TransactionRemovedFromMempool(block.vtx[index]); } for (const CTransactionRef& ptx : vtxConflicted) { TransactionRemovedFromMempool(ptx); } - - m_last_block_processed = block_hash; } -void CWallet::BlockDisconnected(const CBlock& block) { +void CWallet::BlockDisconnected(const CBlock& block, int height) +{ auto locked_chain = chain().lock(); LOCK(cs_wallet); @@ -1430,8 +1094,11 @@ void CWallet::BlockDisconnected(const CBlock& block) { // be unconfirmed, whether or not the transaction is added back to the mempool. // User may have to call abandontransaction again. It may be addressed in the // future with a stickier abandoned state or even removing abandontransaction call. + m_last_block_processed_height = height - 1; + m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - SyncTransaction(ptx, CWalletTx::Status::UNCONFIRMED, {} /* block hash */, 0 /* position in block */); + CWalletTx::Confirmation confirm(CWalletTx::Status::UNCONFIRMED, /* block_height */ 0, {}, /* nIndex */ 0); + SyncTransaction(ptx, confirm); } } @@ -1448,7 +1115,7 @@ void CWallet::BlockUntilSyncedToCurrentChain() { // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); - chain().waitForNotificationsIfNewBlocksConnected(last_block_hash); + chain().waitForNotificationsIfTipChanged(last_block_hash); } @@ -1487,7 +1154,21 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const isminetype CWallet::IsMine(const CTxOut& txout) const { - return ::IsMine(*this, txout.scriptPubKey); + return IsMine(txout.scriptPubKey); +} + +isminetype CWallet::IsMine(const CTxDestination& dest) const +{ + return IsMine(GetScriptForDestination(dest)); +} + +isminetype CWallet::IsMine(const CScript& script) const +{ + isminetype result = ISMINE_NO; + if (auto spk_man = m_spk_man.get()) { + result = spk_man->IsMine(script); + } + return result; } CAmount CWallet::GetCredit(const CTxOut& txout, const isminefilter& filter) const @@ -1511,7 +1192,7 @@ bool CWallet::IsChange(const CScript& script) const // a better way of identifying which outputs are 'the send' and which are // 'the change' will need to be implemented (maybe extend CWalletTx to remember // which output, if any, was change). - if (::IsMine(*this, script)) + if (IsMine(script)) { CTxDestination address; if (!ExtractDestination(script, address)) @@ -1601,92 +1282,24 @@ CAmount CWallet::GetChange(const CTransaction& tx) const return nChange; } -CPubKey CWallet::GenerateNewSeed() -{ - assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - CKey key; - key.MakeNewKey(true); - return DeriveNewSeed(key); -} - -CPubKey CWallet::DeriveNewSeed(const CKey& key) -{ - int64_t nCreationTime = GetTime(); - CKeyMetadata metadata(nCreationTime); - - // calculate the seed - CPubKey seed = key.GetPubKey(); - assert(key.VerifyPubKey(seed)); - - // set the hd keypath to "s" -> Seed, refers the seed to itself - metadata.hdKeypath = "s"; - metadata.has_key_origin = false; - metadata.hd_seed_id = seed.GetID(); - - { - LOCK(cs_wallet); - - // mem store the metadata - mapKeyMetadata[seed.GetID()] = metadata; - - // write the key&metadata to the database - if (!AddKeyPubKey(key, seed)) - throw std::runtime_error(std::string(__func__) + ": AddKeyPubKey failed"); - } - - return seed; -} - -void CWallet::SetHDSeed(const CPubKey& seed) -{ - LOCK(cs_wallet); - // store the keyid (hash160) together with - // the child index counter in the database - // as a hdchain object - CHDChain newHdChain; - newHdChain.nVersion = CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; - newHdChain.seed_id = seed.GetID(); - SetHDChain(newHdChain, false); - NotifyCanGetAddressesChanged(); - UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); -} - -void CWallet::SetHDChain(const CHDChain& chain, bool memonly) -{ - LOCK(cs_wallet); - if (!memonly && !WalletBatch(*database).WriteHDChain(chain)) - throw std::runtime_error(std::string(__func__) + ": writing chain failed"); - - hdChain = chain; -} - bool CWallet::IsHDEnabled() const { - return !hdChain.seed_id.IsNull(); -} - -bool CWallet::CanGenerateKeys() -{ - // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) - LOCK(cs_wallet); - return IsHDEnabled() || !CanSupportFeature(FEATURE_HD); + bool result = true; + if (auto spk_man = m_spk_man.get()) { + result &= spk_man->IsHDEnabled(); + } + return result; } bool CWallet::CanGetAddresses(bool internal) { - LOCK(cs_wallet); - // Check if the keypool has keys - bool keypool_has_keys; - if (internal && CanSupportFeature(FEATURE_HD_SPLIT)) { - keypool_has_keys = setInternalKeyPool.size() > 0; - } else { - keypool_has_keys = KeypoolCountExternalKeys() > 0; - } - // If the keypool doesn't have keys, check if we can generate them - if (!keypool_has_keys) { - return CanGenerateKeys(); + { + auto spk_man = m_spk_man.get(); + if (spk_man && spk_man->CanGetAddresses(internal)) { + return true; + } } - return keypool_has_keys; + return false; } void CWallet::SetWalletFlag(uint64_t flags) @@ -1711,6 +1324,11 @@ void CWallet::UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } +void CWallet::UnsetBlankWalletFlag(WalletBatch& batch) +{ + UnsetWalletFlagWithDB(batch, WALLET_FLAG_BLANK_WALLET); +} + bool CWallet::IsWalletFlagSet(uint64_t flag) const { return (m_wallet_flags & flag); @@ -1745,7 +1363,9 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(*this, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { + const SigningProvider* provider = GetSigningProvider(); + + if (!ProduceSignature(*provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } UpdateInput(tx_in, sigdata); @@ -1770,97 +1390,53 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> bool CWallet::ImportScripts(const std::set<CScript> scripts, int64_t timestamp) { - WalletBatch batch(*database); - for (const auto& entry : scripts) { - CScriptID id(entry); - if (HaveCScript(id)) { - WalletLogPrintf("Already have script %s, skipping\n", HexStr(entry)); - continue; - } - if (!AddCScriptWithDB(batch, entry)) { - return false; - } - - if (timestamp > 0) { - m_script_metadata[CScriptID(entry)].nCreateTime = timestamp; - } - } - if (timestamp > 0) { - UpdateTimeFirstKey(timestamp); + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; } - - return true; + AssertLockHeld(spk_man->cs_wallet); + return spk_man->ImportScripts(scripts, timestamp); } bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) { - WalletBatch batch(*database); - for (const auto& entry : privkey_map) { - const CKey& key = entry.second; - CPubKey pubkey = key.GetPubKey(); - const CKeyID& id = entry.first; - assert(key.VerifyPubKey(pubkey)); - // Skip if we already have the key - if (HaveKey(id)) { - WalletLogPrintf("Already have key with pubkey %s, skipping\n", HexStr(pubkey)); - continue; - } - mapKeyMetadata[id].nCreateTime = timestamp; - // If the private key is not present in the wallet, insert it. - if (!AddKeyPubKeyWithDB(batch, key, pubkey)) { - return false; - } - UpdateTimeFirstKey(timestamp); + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; } - return true; + AssertLockHeld(spk_man->cs_wallet); + return spk_man->ImportPrivKeys(privkey_map, timestamp); } bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) { - WalletBatch batch(*database); - for (const auto& entry : key_origins) { - AddKeyOriginWithDB(batch, entry.second.first, entry.second.second); - } - for (const CKeyID& id : ordered_pubkeys) { - auto entry = pubkey_map.find(id); - if (entry == pubkey_map.end()) { - continue; - } - const CPubKey& pubkey = entry->second; - CPubKey temp; - if (GetPubKey(id, temp)) { - // Already have pubkey, skipping - WalletLogPrintf("Already have pubkey %s, skipping\n", HexStr(temp)); - continue; - } - if (!AddWatchOnlyWithDB(batch, GetScriptForRawPubKey(pubkey), timestamp)) { - return false; - } - mapKeyMetadata[id].nCreateTime = timestamp; - - // Add to keypool only works with pubkeys - if (add_keypool) { - AddKeypoolPubkeyWithDB(pubkey, internal, batch); - NotifyCanGetAddressesChanged(); - } + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; } - return true; + AssertLockHeld(spk_man->cs_wallet); + return spk_man->ImportPubKeys(ordered_pubkeys, pubkey_map, key_origins, add_keypool, internal, timestamp); } bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) { - WalletBatch batch(*database); - for (const CScript& script : script_pub_keys) { - if (!have_solving_data || !::IsMine(*this, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated - if (!AddWatchOnlyWithDB(batch, script, timestamp)) { - return false; + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return false; + } + AssertLockHeld(spk_man->cs_wallet); + if (!spk_man->ImportScriptPubKeys(script_pub_keys, have_solving_data, timestamp)) { + return false; + } + if (apply_label) { + WalletBatch batch(*database); + for (const CScript& script : script_pub_keys) { + CTxDestination dest; + ExtractDestination(script, dest); + if (IsValidDestination(dest)) { + SetAddressBookWithDB(batch, dest, label, "receive"); } } - CTxDestination dest; - ExtractDestination(script, dest); - if (apply_label && IsValidDestination(dest)) { - SetAddressBookWithDB(batch, dest, label, "receive"); - } } return true; } @@ -2068,7 +1644,8 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], CWalletTx::Status::CONFIRMED, block_hash, posInBlock, fUpdate); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *block_height, block_hash, posInBlock); + SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -2116,7 +1693,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc return result; } -void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) +void CWallet::ReacceptWalletTransactions() { // If transactions aren't being broadcasted, don't let them into local mempool either if (!fBroadcastTransactions) @@ -2129,7 +1706,7 @@ void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -2140,11 +1717,11 @@ void CWallet::ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); std::string unused_err_string; - wtx.SubmitMemoryPoolAndRelay(unused_err_string, false, locked_chain); + wtx.SubmitMemoryPoolAndRelay(unused_err_string, false); } } -bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, interfaces::Chain::Lock& locked_chain) +bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay) { // Can't relay if wallet is not broadcasting if (!pwallet->GetBroadcastTransactions()) return false; @@ -2154,7 +1731,7 @@ bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, in // cause log spam. if (IsCoinBase()) return false; // Don't try to submit conflicted or confirmed transactions. - if (GetDepthInMainChain(locked_chain) != 0) return false; + if (GetDepthInMainChain() != 0) return false; // Submit transaction to mempool for relay pwallet->WalletLogPrintf("Submitting wtx %s to mempool for relay\n", GetHash().ToString()); @@ -2208,10 +1785,10 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const return debit; } -CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const +CAmount CWalletTx::GetCredit(const isminefilter& filter) const { // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase(locked_chain)) + if (IsImmatureCoinBase()) return 0; CAmount credit = 0; @@ -2225,16 +1802,16 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine return credit; } -CAmount CWalletTx::GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache) const +CAmount CWalletTx::GetImmatureCredit(bool fUseCache) const { - if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { + if (IsImmatureCoinBase() && IsInMainChain()) { return GetCachableAmount(IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); } return 0; } -CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache, const isminefilter& filter) const +CAmount CWalletTx::GetAvailableCredit(bool fUseCache, const isminefilter& filter) const { if (pwallet == nullptr) return 0; @@ -2243,7 +1820,7 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo bool allow_cache = (filter & ISMINE_ALL) && (filter & ISMINE_ALL) != ISMINE_ALL; // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase(locked_chain)) + if (IsImmatureCoinBase()) return 0; if (fUseCache && allow_cache && m_amounts[AVAILABLE_CREDIT].m_cached[filter]) { @@ -2255,7 +1832,7 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo uint256 hashTx = GetHash(); for (unsigned int i = 0; i < tx->vout.size(); i++) { - if (!pwallet->IsSpent(locked_chain, hashTx, i) && (allow_used_addresses || !pwallet->IsUsedDestination(hashTx, i))) { + if (!pwallet->IsSpent(hashTx, i) && (allow_used_addresses || !pwallet->IsUsedDestination(hashTx, i))) { const CTxOut &txout = tx->vout[i]; nCredit += pwallet->GetCredit(txout, filter); if (!MoneyRange(nCredit)) @@ -2270,9 +1847,9 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo return nCredit; } -CAmount CWalletTx::GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache) const +CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool fUseCache) const { - if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { + if (IsImmatureCoinBase() && IsInMainChain()) { return GetCachableAmount(IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); } @@ -2295,32 +1872,37 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { + std::set<uint256> s; + return IsTrusted(locked_chain, s); +} + +bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const +{ // Quick answer in most cases - if (!locked_chain.checkFinalTx(*tx)) { - return false; - } - int nDepth = GetDepthInMainChain(locked_chain); - if (nDepth >= 1) - return true; - if (nDepth < 0) - return false; - if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) // using wtx's cached debit - return false; + if (!locked_chain.checkFinalTx(*tx)) return false; + int nDepth = GetDepthInMainChain(); + if (nDepth >= 1) return true; + if (nDepth < 0) return false; + // using wtx's cached debit + if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) return false; // Don't trust unconfirmed transactions from us unless they are in the mempool. - if (!InMempool()) - return false; + if (!InMempool()) return false; // Trusted if all inputs are from us and are in the mempool: for (const CTxIn& txin : tx->vin) { // Transactions not sent by us: not trusted const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); - if (parent == nullptr) - return false; + if (parent == nullptr) return false; const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; - if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) - return false; + // Check that this specific input being spent is trusted + if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; + // If we've already trusted this parent, continue + if (trusted_parents.count(parent->GetHash())) continue; + // Recurse to check that the parent is also trusted + if (!parent->IsTrusted(locked_chain, trusted_parents)) return false; + trusted_parents.insert(parent->GetHash()); } return true; } @@ -2374,7 +1956,7 @@ void CWallet::ResendWalletTransactions() // any confirmed or conflicting txs. if (wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; std::string unused_err_string; - if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true, *locked_chain)) ++submitted_tx_count; + if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true)) ++submitted_tx_count; } } // locked_chain and cs_wallet @@ -2406,13 +1988,14 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons { auto locked_chain = chain().lock(); LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(*locked_chain)}; - const int tx_depth{wtx.GetDepthInMainChain(*locked_chain)}; - const CAmount tx_credit_mine{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(*locked_chain, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; + const bool is_trusted{wtx.IsTrusted(*locked_chain, trusted_parents)}; + const int tx_depth{wtx.GetDepthInMainChain()}; + const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; @@ -2421,8 +2004,8 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += wtx.GetImmatureCredit(*locked_chain); - ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(*locked_chain); + ret.m_mine_immature += wtx.GetImmatureCredit(); + ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(); } } return ret; @@ -2456,6 +2039,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const int min_depth = {coinControl ? coinControl->m_min_depth : DEFAULT_MIN_DEPTH}; const int max_depth = {coinControl ? coinControl->m_max_depth : DEFAULT_MAX_DEPTH}; + std::set<uint256> trusted_parents; for (const auto& entry : mapWallet) { const uint256& wtxid = entry.first; @@ -2465,10 +2049,10 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - if (wtx.IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase()) continue; - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < 0) continue; @@ -2477,7 +2061,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(locked_chain); + bool safeTx = wtx.IsTrusted(locked_chain, trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -2528,7 +2112,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< if (IsLockedCoin(entry.first, i)) continue; - if (IsSpent(locked_chain, wtxid, i)) + if (IsSpent(wtxid, i)) continue; isminetype mine = IsMine(wtx.tx->vout[i]); @@ -2541,7 +2125,9 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - bool solvable = IsSolvable(*this, wtx.tx->vout[i].scriptPubKey); + const SigningProvider* provider = GetSigningProvider(); + + bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); @@ -2585,7 +2171,7 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Ch for (const COutPoint& output : lockedCoins) { auto it = mapWallet.find(output.hash); if (it != mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(locked_chain); + int depth = it->second.GetDepthInMainChain(); if (depth >= 0 && output.n < it->second.tx->vout.size() && IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { CTxDestination address; @@ -2775,7 +2361,13 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const CScript& scriptPubKey = mi->second.tx->vout[input.prevout.n].scriptPubKey; const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; SignatureData sigdata; - if (!ProduceSignature(*this, MutableTransactionSignatureCreator(&tx, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { + + const SigningProvider* provider = GetSigningProvider(); + if (!provider) { + return false; + } + + if (!ProduceSignature(*provider, MutableTransactionSignatureCreator(&tx, nIn, amount, SIGHASH_ALL), scriptPubKey, sigdata)) { return false; } UpdateInput(input, sigdata); @@ -3233,7 +2825,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(*this, MutableTransactionSignatureCreator(&txNew, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) + const SigningProvider* provider = GetSigningProvider(); + if (!provider || !ProduceSignature(*provider, MutableTransactionSignatureCreator(&txNew, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed").translated; return false; @@ -3284,51 +2877,44 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std return true; } -/** - * Call after CreateTransaction unless you want to abort - */ -bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CValidationState& state) +void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { - { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); + auto locked_chain = chain().lock(); + LOCK(cs_wallet); - CWalletTx wtxNew(this, std::move(tx)); - wtxNew.mapValue = std::move(mapValue); - wtxNew.vOrderForm = std::move(orderForm); - wtxNew.fTimeReceivedIsTxTime = true; - wtxNew.fFromMe = true; + CWalletTx wtxNew(this, std::move(tx)); + wtxNew.mapValue = std::move(mapValue); + wtxNew.vOrderForm = std::move(orderForm); + wtxNew.fTimeReceivedIsTxTime = true; + wtxNew.fFromMe = true; - WalletLogPrintf("CommitTransaction:\n%s", wtxNew.tx->ToString()); /* Continued */ - { + WalletLogPrintf("CommitTransaction:\n%s", wtxNew.tx->ToString()); /* Continued */ - // Add tx to wallet, because if it has change it's also ours, - // otherwise just for transaction history. - AddToWallet(wtxNew); + // Add tx to wallet, because if it has change it's also ours, + // otherwise just for transaction history. + AddToWallet(wtxNew); - // Notify that old coins are spent - for (const CTxIn& txin : wtxNew.tx->vin) - { - CWalletTx &coin = mapWallet.at(txin.prevout.hash); - coin.BindWallet(this); - NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED); - } - } + // Notify that old coins are spent + for (const CTxIn& txin : wtxNew.tx->vin) { + CWalletTx &coin = mapWallet.at(txin.prevout.hash); + coin.BindWallet(this); + NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED); + } - // Get the inserted-CWalletTx from mapWallet so that the - // fInMempool flag is cached properly - CWalletTx& wtx = mapWallet.at(wtxNew.GetHash()); + // Get the inserted-CWalletTx from mapWallet so that the + // fInMempool flag is cached properly + CWalletTx& wtx = mapWallet.at(wtxNew.GetHash()); - if (fBroadcastTransactions) - { - std::string err_string; - if (!wtx.SubmitMemoryPoolAndRelay(err_string, true, *locked_chain)) { - WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string); - // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. - } - } + if (!fBroadcastTransactions) { + // Don't submit tx to the mempool + return; + } + + std::string err_string; + if (!wtx.SubmitMemoryPoolAndRelay(err_string, true)) { + WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string); + // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } - return true; } DBErrors CWallet::LoadWallet(bool& fFirstRunRet) @@ -3346,12 +2932,9 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { if (database->Rewrite("\x04pool")) { - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + if (auto spk_man = m_spk_man.get()) { + spk_man->RewriteDB(); + } } } @@ -3383,12 +2966,9 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { if (database->Rewrite("\x04pool")) { - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + if (auto spk_man = m_spk_man.get()) { + spk_man->RewriteDB(); + } } } @@ -3407,13 +2987,9 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { if (database->Rewrite("\x04pool")) { - LOCK(cs_wallet); - setInternalKeyPool.clear(); - setExternalKeyPool.clear(); - m_pool_key_to_index.clear(); - // Note: can't top-up keypool here, because wallet is locked. - // User will be prompted to unlock wallet the next operation - // that requires a new key. + if (auto spk_man = m_spk_man.get()) { + spk_man->RewriteDB(); + } } } @@ -3434,7 +3010,7 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add if (!strPurpose.empty()) /* update purpose only if requested */ mapAddressBook[address].purpose = strPurpose; } - NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, + NotifyAddressBookChanged(this, address, strName, IsMine(address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose)) return false; @@ -3461,258 +3037,66 @@ bool CWallet::DelAddressBook(const CTxDestination& address) mapAddressBook.erase(address); } - NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); + NotifyAddressBookChanged(this, address, "", IsMine(address) != ISMINE_NO, "", CT_DELETED); WalletBatch(*database).ErasePurpose(EncodeDestination(address)); return WalletBatch(*database).EraseName(EncodeDestination(address)); } -/** - * Mark old keypool keys as used, - * and generate all new keys - */ -bool CWallet::NewKeyPool() -{ - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - return false; - } - { - LOCK(cs_wallet); - WalletBatch batch(*database); - - for (const int64_t nIndex : setInternalKeyPool) { - batch.ErasePool(nIndex); - } - setInternalKeyPool.clear(); - - for (const int64_t nIndex : setExternalKeyPool) { - batch.ErasePool(nIndex); - } - setExternalKeyPool.clear(); - - for (const int64_t nIndex : set_pre_split_keypool) { - batch.ErasePool(nIndex); - } - set_pre_split_keypool.clear(); - - m_pool_key_to_index.clear(); - - if (!TopUpKeyPool()) { - return false; - } - WalletLogPrintf("CWallet::NewKeyPool rewrote keypool\n"); - } - return true; -} - size_t CWallet::KeypoolCountExternalKeys() { AssertLockHeld(cs_wallet); - return setExternalKeyPool.size() + set_pre_split_keypool.size(); -} -void CWallet::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) -{ - AssertLockHeld(cs_wallet); - if (keypool.m_pre_split) { - set_pre_split_keypool.insert(nIndex); - } else if (keypool.fInternal) { - setInternalKeyPool.insert(nIndex); - } else { - setExternalKeyPool.insert(nIndex); + unsigned int count = 0; + if (auto spk_man = m_spk_man.get()) { + AssertLockHeld(spk_man->cs_wallet); + count += spk_man->KeypoolCountExternalKeys(); } - m_max_keypool_index = std::max(m_max_keypool_index, nIndex); - m_pool_key_to_index[keypool.vchPubKey.GetID()] = nIndex; - // If no metadata exists yet, create a default with the pool key's - // creation time. Note that this may be overwritten by actually - // stored metadata for that key later, which is fine. - CKeyID keyid = keypool.vchPubKey.GetID(); - if (mapKeyMetadata.count(keyid) == 0) - mapKeyMetadata[keyid] = CKeyMetadata(keypool.nTime); + return count; } -bool CWallet::TopUpKeyPool(unsigned int kpSize) +unsigned int CWallet::GetKeyPoolSize() const { - if (!CanGenerateKeys()) { - return false; - } - { - LOCK(cs_wallet); - - if (IsLocked()) return false; - - // Top up key pool - unsigned int nTargetSize; - if (kpSize > 0) - nTargetSize = kpSize; - else - nTargetSize = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); - - // count amount of available keys (internal, external) - // make sure the keypool of external and internal keys fits the user selected target (-keypool) - int64_t missingExternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setExternalKeyPool.size(), (int64_t) 0); - int64_t missingInternal = std::max(std::max((int64_t) nTargetSize, (int64_t) 1) - (int64_t)setInternalKeyPool.size(), (int64_t) 0); - - if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) - { - // don't create extra internal keys - missingInternal = 0; - } - bool internal = false; - WalletBatch batch(*database); - for (int64_t i = missingInternal + missingExternal; i--;) - { - if (i < missingInternal) { - internal = true; - } - - CPubKey pubkey(GenerateNewKey(batch, internal)); - AddKeypoolPubkeyWithDB(pubkey, internal, batch); - } - if (missingInternal + missingExternal > 0) { - WalletLogPrintf("keypool added %d keys (%d internal), size=%u (%u internal)\n", missingInternal + missingExternal, missingInternal, setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(), setInternalKeyPool.size()); - } - } - NotifyCanGetAddressesChanged(); - return true; -} - -void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) -{ - LOCK(cs_wallet); - assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? - int64_t index = ++m_max_keypool_index; - if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { - throw std::runtime_error(std::string(__func__) + ": writing imported pubkey failed"); - } - if (internal) { - setInternalKeyPool.insert(index); - } else { - setExternalKeyPool.insert(index); - } - m_pool_key_to_index[pubkey.GetID()] = index; -} - -bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal) -{ - nIndex = -1; - keypool.vchPubKey = CPubKey(); - { - LOCK(cs_wallet); - - TopUpKeyPool(); - - bool fReturningInternal = fRequestedInternal; - fReturningInternal &= (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) || IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - bool use_split_keypool = set_pre_split_keypool.empty(); - std::set<int64_t>& setKeyPool = use_split_keypool ? (fReturningInternal ? setInternalKeyPool : setExternalKeyPool) : set_pre_split_keypool; - - // Get the oldest key - if (setKeyPool.empty()) { - return false; - } - - WalletBatch batch(*database); - - auto it = setKeyPool.begin(); - nIndex = *it; - setKeyPool.erase(it); - if (!batch.ReadPool(nIndex, keypool)) { - throw std::runtime_error(std::string(__func__) + ": read failed"); - } - CPubKey pk; - if (!GetPubKey(keypool.vchPubKey.GetID(), pk)) { - throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); - } - // If the key was pre-split keypool, we don't care about what type it is - if (use_split_keypool && keypool.fInternal != fReturningInternal) { - throw std::runtime_error(std::string(__func__) + ": keypool entry misclassified"); - } - if (!keypool.vchPubKey.IsValid()) { - throw std::runtime_error(std::string(__func__) + ": keypool entry invalid"); - } - - m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); - WalletLogPrintf("keypool reserve %d\n", nIndex); - } - NotifyCanGetAddressesChanged(); - return true; -} - -void CWallet::KeepKey(int64_t nIndex) -{ - // Remove from key pool - WalletBatch batch(*database); - batch.ErasePool(nIndex); - WalletLogPrintf("keypool keep %d\n", nIndex); -} + AssertLockHeld(cs_wallet); -void CWallet::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey) -{ - // Return to key pool - { - LOCK(cs_wallet); - if (fInternal) { - setInternalKeyPool.insert(nIndex); - } else if (!set_pre_split_keypool.empty()) { - set_pre_split_keypool.insert(nIndex); - } else { - setExternalKeyPool.insert(nIndex); - } - m_pool_key_to_index[pubkey.GetID()] = nIndex; - NotifyCanGetAddressesChanged(); + unsigned int count = 0; + if (auto spk_man = m_spk_man.get()) { + count += spk_man->GetKeyPoolSize(); } - WalletLogPrintf("keypool return %d\n", nIndex); + return count; } -bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) +bool CWallet::TopUpKeyPool(unsigned int kpSize) { - if (!CanGetAddresses(internal)) { - return false; + bool res = true; + if (auto spk_man = m_spk_man.get()) { + res &= spk_man->TopUp(kpSize); } - - CKeyPool keypool; - { - LOCK(cs_wallet); - int64_t nIndex; - if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - if (IsLocked()) return false; - WalletBatch batch(*database); - result = GenerateNewKey(batch, internal); - return true; - } - KeepKey(nIndex); - result = keypool.vchPubKey; - } - return true; + return res; } bool CWallet::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error) { LOCK(cs_wallet); error.clear(); - - TopUpKeyPool(); - - // Generate a new key that is added to wallet - CPubKey new_key; - if (!GetKeyFromPool(new_key)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; - return false; + bool result = false; + auto spk_man = m_spk_man.get(); + if (spk_man) { + result = spk_man->GetNewDestination(type, dest, error); + } + if (result) { + SetAddressBook(dest, label, "receive"); } - LearnRelatedScripts(new_key, type); - dest = GetDestinationForKey(new_key, type); - SetAddressBook(dest, label, "receive"); - return true; + return result; } bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error) { error.clear(); - TopUpKeyPool(); + m_spk_man->TopUp(); ReserveDestination reservedest(this); if (!reservedest.GetReservedDestination(type, dest, true)) { @@ -3724,35 +3108,12 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des return true; } -static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, WalletBatch& batch) { - if (setKeyPool.empty()) { - return GetTime(); - } - - CKeyPool keypool; - int64_t nIndex = *(setKeyPool.begin()); - if (!batch.ReadPool(nIndex, keypool)) { - throw std::runtime_error(std::string(__func__) + ": read oldest key in keypool failed"); - } - assert(keypool.vchPubKey.IsValid()); - return keypool.nTime; -} - int64_t CWallet::GetOldestKeyPoolTime() { - LOCK(cs_wallet); - - WalletBatch batch(*database); - - // load oldest key from keypool, get time and return - int64_t oldestKey = GetOldestKeyTimeInPool(setExternalKeyPool, batch); - if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) { - oldestKey = std::max(GetOldestKeyTimeInPool(setInternalKeyPool, batch), oldestKey); - if (!set_pre_split_keypool.empty()) { - oldestKey = std::max(GetOldestKeyTimeInPool(set_pre_split_keypool, batch), oldestKey); - } + int64_t oldestKey = std::numeric_limits<int64_t>::max(); + if (auto spk_man = m_spk_man.get()) { + oldestKey = spk_man->GetOldestKeyPoolTime(); } - return oldestKey; } @@ -3762,17 +3123,18 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: { LOCK(cs_wallet); + std::set<uint256> trusted_parents; for (const auto& walletEntry : mapWallet) { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(locked_chain)) + if (!wtx.IsTrusted(locked_chain, trusted_parents)) continue; - if (wtx.IsImmatureCoinBase(locked_chain)) + if (wtx.IsImmatureCoinBase()) continue; - int nDepth = wtx.GetDepthInMainChain(locked_chain); + int nDepth = wtx.GetDepthInMainChain(); if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) continue; @@ -3784,7 +3146,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(locked_chain, walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; + CAmount n = IsSpent(walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; if (!balances.count(addr)) balances[addr] = 0; @@ -3905,6 +3267,11 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co bool ReserveDestination::GetReservedDestination(const OutputType type, CTxDestination& dest, bool internal) { + m_spk_man = pwallet->GetLegacyScriptPubKeyMan(); + if (!m_spk_man) { + return false; + } + if (!pwallet->CanGetAddresses(internal)) { return false; } @@ -3912,14 +3279,14 @@ bool ReserveDestination::GetReservedDestination(const OutputType type, CTxDestin if (nIndex == -1) { CKeyPool keypool; - if (!pwallet->ReserveKeyFromKeyPool(nIndex, keypool, internal)) { + if (!m_spk_man->GetReservedDestination(type, internal, nIndex, keypool)) { return false; } vchPubKey = keypool.vchPubKey; fInternal = keypool.fInternal; } assert(vchPubKey.IsValid()); - pwallet->LearnRelatedScripts(vchPubKey, type); + m_spk_man->LearnRelatedScripts(vchPubKey, type); address = GetDestinationForKey(vchPubKey, type); dest = address; return true; @@ -3928,7 +3295,7 @@ bool ReserveDestination::GetReservedDestination(const OutputType type, CTxDestin void ReserveDestination::KeepDestination() { if (nIndex != -1) - pwallet->KeepKey(nIndex); + m_spk_man->KeepDestination(nIndex); nIndex = -1; vchPubKey = CPubKey(); address = CNoDestination(); @@ -3937,37 +3304,13 @@ void ReserveDestination::KeepDestination() void ReserveDestination::ReturnDestination() { if (nIndex != -1) { - pwallet->ReturnKey(nIndex, fInternal, vchPubKey); + m_spk_man->ReturnDestination(nIndex, fInternal, vchPubKey); } nIndex = -1; vchPubKey = CPubKey(); address = CNoDestination(); } -void CWallet::MarkReserveKeysAsUsed(int64_t keypool_id) -{ - AssertLockHeld(cs_wallet); - bool internal = setInternalKeyPool.count(keypool_id); - if (!internal) assert(setExternalKeyPool.count(keypool_id) || set_pre_split_keypool.count(keypool_id)); - std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); - auto it = setKeyPool->begin(); - - WalletBatch batch(*database); - while (it != std::end(*setKeyPool)) { - const int64_t& index = *(it); - if (index > keypool_id) break; // set*KeyPool is ordered - - CKeyPool keypool; - if (batch.ReadPool(index, keypool)) { //TODO: This should be unnecessary - m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); - } - LearnAllRelatedScripts(keypool.vchPubKey); - batch.ErasePool(index); - WalletLogPrintf("keypool index %d removed\n", index); - it = setKeyPool->erase(it); - } -} - void CWallet::LockCoin(const COutPoint& output) { AssertLockHeld(cs_wallet); @@ -4010,8 +3353,12 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C AssertLockHeld(cs_wallet); mapKeyBirth.clear(); + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + assert(spk_man != nullptr); + AssertLockHeld(spk_man->cs_wallet); + // get birth times for keys with metadata - for (const auto& entry : mapKeyMetadata) { + for (const auto& entry : spk_man->mapKeyMetadata) { if (entry.second.nCreateTime) { mapKeyBirth[entry.first] = entry.second.nCreateTime; } @@ -4021,7 +3368,7 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C const Optional<int> tip_height = locked_chain.getHeight(); const int max_height = tip_height && *tip_height > 144 ? *tip_height - 144 : 0; // the tip can be reorganized; use a 144-block safety margin std::map<CKeyID, int> mapKeyFirstBlock; - for (const CKeyID &keyid : GetKeys()) { + for (const CKeyID &keyid : spk_man->GetKeys()) { if (mapKeyBirth.count(keyid) == 0) mapKeyFirstBlock[keyid] = max_height; } @@ -4038,7 +3385,7 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C // ... which are already in a block for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs - for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *this)) { + for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { // ... and all their affected keys std::map<CKeyID, int>::iterator rit = mapKeyFirstBlock.find(keyid); if (rit != mapKeyFirstBlock.end() && *height < rit->second) @@ -4113,20 +3460,20 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const return nTimeSmart; } -bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, const std::string &value) +bool CWallet::AddDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key, const std::string &value) { if (boost::get<CNoDestination>(&dest)) return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - return WalletBatch(*database).WriteDestData(EncodeDestination(dest), key, value); + return batch.WriteDestData(EncodeDestination(dest), key, value); } -bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) +bool CWallet::EraseDestData(WalletBatch& batch, const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - return WalletBatch(*database).EraseDestData(EncodeDestination(dest), key); + return batch.EraseDestData(EncodeDestination(dest), key); } void CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -4163,24 +3510,6 @@ std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const return values; } -void CWallet::MarkPreSplitKeys() -{ - WalletBatch batch(*database); - for (auto it = setExternalKeyPool.begin(); it != setExternalKeyPool.end();) { - int64_t index = *it; - CKeyPool keypool; - if (!batch.ReadPool(index, keypool)) { - throw std::runtime_error(std::string(__func__) + ": read keypool entry failed"); - } - keypool.m_pre_split = true; - if (!batch.WritePool(index, keypool)) { - throw std::runtime_error(std::string(__func__) + ": writing modified keypool entry failed"); - } - set_pre_split_keypool.insert(index); - it = setExternalKeyPool.erase(it); - } -} - bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bool salvage_wallet, std::string& error_string, std::vector<std::string>& warnings) { // Do some checking on wallet path. It should be either a: @@ -4321,31 +3650,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } - bool hd_upgrade = false; - bool split_upgrade = false; - if (walletInstance->CanSupportFeature(FEATURE_HD) && !walletInstance->IsHDEnabled()) { - walletInstance->WalletLogPrintf("Upgrading wallet to HD\n"); - walletInstance->SetMinVersion(FEATURE_HD); - - // generate a new master key - CPubKey masterPubKey = walletInstance->GenerateNewSeed(); - walletInstance->SetHDSeed(masterPubKey); - hd_upgrade = true; - } - // Upgrade to HD chain split if necessary - if (walletInstance->CanSupportFeature(FEATURE_HD_SPLIT)) { - walletInstance->WalletLogPrintf("Upgrading wallet to use HD chain split\n"); - walletInstance->SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); - split_upgrade = FEATURE_HD_SPLIT > prev_version; - } - // Mark all keys currently in the keypool as pre-split - if (split_upgrade) { - walletInstance->MarkPreSplitKeys(); - } - // Regenerate the keypool if upgraded to HD - if (hd_upgrade) { - if (!walletInstance->TopUpKeyPool()) { - error = _("Unable to generate keys").translated; + if (auto spk_man = walletInstance->m_spk_man.get()) { + std::string error; + if (!spk_man->Upgrade(prev_version, error)) { + chain.initError(error); return nullptr; } } @@ -4358,15 +3666,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetWalletFlags(wallet_creation_flags, false); if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { - // generate a new seed - CPubKey seed = walletInstance->GenerateNewSeed(); - walletInstance->SetHDSeed(seed); - } - - // Top up the keypool - if (walletInstance->CanGenerateKeys() && !walletInstance->TopUpKeyPool()) { - error = _("Unable to generate initial keys").translated; - return nullptr; + if (auto spk_man = walletInstance->m_spk_man.get()) { + if (!spk_man->SetupGeneration()) { + error = _("Unable to generate initial keys").translated; + return nullptr; + } + } } auto locked_chain = chain.lock(); @@ -4376,9 +3681,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - LOCK(walletInstance->cs_KeyStore); - if (!walletInstance->mapKeys.empty() || !walletInstance->mapCryptedKeys.empty()) { - warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + if (walletInstance->m_spk_man) { + if (walletInstance->m_spk_man->HavePrivateKeys()) { + warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + } } } @@ -4499,8 +3805,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const Optional<int> tip_height = locked_chain->getHeight(); if (tip_height) { walletInstance->m_last_block_processed = locked_chain->getBlockHash(*tip_height); + walletInstance->m_last_block_processed_height = *tip_height; } else { walletInstance->m_last_block_processed.SetNull(); + walletInstance->m_last_block_processed_height = -1; } if (tip_height && *tip_height != rescan_height) @@ -4528,8 +3836,13 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - if (walletInstance->nTimeFirstKey) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height, nullptr)) { + Optional<int64_t> time_first_key; + if (auto spk_man = walletInstance->m_spk_man.get()) { + int64_t time = spk_man->GetTimeFirstKey(); + if (!time_first_key || time < *time_first_key) time_first_key = time; + } + if (time_first_key) { + if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } @@ -4597,7 +3910,7 @@ void CWallet::postInitProcess() // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded - ReacceptWalletTransactions(*locked_chain); + ReacceptWalletTransactions(); // Update wallet transactions with current mempool transactions. chain().requestMempoolTransactions(*this); @@ -4623,55 +3936,28 @@ CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) m_pre_split = false; } -void CWalletTx::SetConf(Status status, const uint256& block_hash, int posInBlock) -{ - // Update tx status - m_confirm.status = status; - - // Update the tx's hashBlock - m_confirm.hashBlock = block_hash; - - // set the position of the transaction in the block - m_confirm.nIndex = posInBlock; -} - -int CWalletTx::GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const +int CWalletTx::GetDepthInMainChain() const { + assert(pwallet != nullptr); + AssertLockHeld(pwallet->cs_wallet); if (isUnconfirmed() || isAbandoned()) return 0; - return locked_chain.getBlockDepth(m_confirm.hashBlock) * (isConflicted() ? -1 : 1); + return (pwallet->GetLastBlockHeight() - m_confirm.block_height + 1) * (isConflicted() ? -1 : 1); } -int CWalletTx::GetBlocksToMaturity(interfaces::Chain::Lock& locked_chain) const +int CWalletTx::GetBlocksToMaturity() const { if (!IsCoinBase()) return 0; - int chain_depth = GetDepthInMainChain(locked_chain); + int chain_depth = GetDepthInMainChain(); assert(chain_depth >= 0); // coinbase tx should not be conflicted return std::max(0, (COINBASE_MATURITY+1) - chain_depth); } -bool CWalletTx::IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const +bool CWalletTx::IsImmatureCoinBase() const { // note GetBlocksToMaturity is 0 for non-coinbase tx - return GetBlocksToMaturity(locked_chain) > 0; -} - -void CWallet::LearnRelatedScripts(const CPubKey& key, OutputType type) -{ - if (key.IsCompressed() && (type == OutputType::P2SH_SEGWIT || type == OutputType::BECH32)) { - CTxDestination witdest = WitnessV0KeyHash(key.GetID()); - CScript witprog = GetScriptForDestination(witdest); - // Make sure the resulting program is solvable. - assert(IsSolvable(*this, witprog)); - AddCScript(witprog); - } -} - -void CWallet::LearnAllRelatedScripts(const CPubKey& key) -{ - // OutputType::P2SH_SEGWIT always adds all necessary scripts for all types. - LearnRelatedScripts(key, OutputType::P2SH_SEGWIT); + return GetBlocksToMaturity() > 0; } std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const { @@ -4704,35 +3990,6 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu return groups; } -bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const -{ - CKeyMetadata meta; - { - LOCK(cs_wallet); - auto it = mapKeyMetadata.find(keyID); - if (it != mapKeyMetadata.end()) { - meta = it->second; - } - } - if (meta.has_key_origin) { - std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); - info.path = meta.key_origin.path; - } else { // Single pubkeys get the master fingerprint of themselves - std::copy(keyID.begin(), keyID.begin() + 4, info.fingerprint); - } - return true; -} - -bool CWallet::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) -{ - LOCK(cs_wallet); - std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); - mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; - mapKeyMetadata[pubkey.GetID()].has_key_origin = true; - mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); - return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); -} - bool CWallet::SetCrypted() { LOCK(cs_KeyStore); @@ -4767,168 +4024,17 @@ bool CWallet::Lock() return true; } -bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) -{ - { - LOCK(cs_KeyStore); - if (!SetCrypted()) - return false; - - bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys - bool keyFail = false; - CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); - for (; mi != mapCryptedKeys.end(); ++mi) - { - const CPubKey &vchPubKey = (*mi).second.first; - const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; - CKey key; - if (!DecryptKey(vMasterKeyIn, vchCryptedSecret, vchPubKey, key)) - { - keyFail = true; - break; - } - keyPass = true; - if (fDecryptionThoroughlyChecked) - break; - } - if (keyPass && keyFail) - { - LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); - throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); - } - if (keyFail || (!keyPass && !accept_no_keys)) - return false; - vMasterKey = vMasterKeyIn; - fDecryptionThoroughlyChecked = true; - } - NotifyStatusChanged(this); - return true; -} - -bool CWallet::HaveKey(const CKeyID &address) const +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan() const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::HaveKey(address); - } - return mapCryptedKeys.count(address) > 0; -} - -bool CWallet::GetKey(const CKeyID &address, CKey& keyOut) const -{ - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::GetKey(address, keyOut); - } - - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - const CPubKey &vchPubKey = (*mi).second.first; - const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; - return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); - } - return false; -} - -bool CWallet::GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const -{ - LOCK(cs_KeyStore); - WatchKeyMap::const_iterator it = mapWatchKeys.find(address); - if (it != mapWatchKeys.end()) { - pubkey_out = it->second; - return true; - } - return false; -} - -bool CWallet::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const -{ - LOCK(cs_KeyStore); - if (!IsCrypted()) { - if (!FillableSigningProvider::GetPubKey(address, vchPubKeyOut)) { - return GetWatchPubKey(address, vchPubKeyOut); - } - return true; - } - - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - vchPubKeyOut = (*mi).second.first; - return true; - } - // Check for watch-only pubkeys - return GetWatchPubKey(address, vchPubKeyOut); -} - -std::set<CKeyID> CWallet::GetKeys() const -{ - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::GetKeys(); - } - std::set<CKeyID> set_address; - for (const auto& mi : mapCryptedKeys) { - set_address.insert(mi.first); - } - return set_address; + return m_spk_man.get(); } -bool CWallet::EncryptKeys(CKeyingMaterial& vMasterKeyIn) +const SigningProvider* CWallet::GetSigningProvider() const { - LOCK(cs_KeyStore); - if (!mapCryptedKeys.empty() || IsCrypted()) - return false; - - fUseCrypto = true; - for (const KeyMap::value_type& mKey : mapKeys) - { - const CKey &key = mKey.second; - CPubKey vchPubKey = key.GetPubKey(); - CKeyingMaterial vchSecret(key.begin(), key.end()); - std::vector<unsigned char> vchCryptedSecret; - if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) - return false; - if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) - return false; - } - mapKeys.clear(); - return true; + return m_spk_man.get(); } -bool CWallet::AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey) +LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return FillableSigningProvider::AddKeyPubKey(key, pubkey); - } - - if (IsLocked()) { - return false; - } - - std::vector<unsigned char> vchCryptedSecret; - CKeyingMaterial vchSecret(key.begin(), key.end()); - if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) { - return false; - } - - if (!AddCryptedKey(pubkey, vchCryptedSecret)) { - return false; - } - return true; -} - - -bool CWallet::AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) -{ - LOCK(cs_KeyStore); - if (!SetCrypted()) { - return false; - } - - mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); - ImplicitlyLearnRelatedKeyScripts(vchPubKey); - return true; + return m_spk_man.get(); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index f9e2230a6f..a6b9c0131a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -18,7 +18,7 @@ #include <validationinterface.h> #include <wallet/coinselection.h> #include <wallet/crypter.h> -#include <wallet/ismine.h> +#include <wallet/scriptpubkeyman.h> #include <wallet/walletdb.h> #include <wallet/walletutil.h> @@ -57,8 +57,6 @@ enum class WalletCreationStatus { WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); -//! Default for -keypool -static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; //! -paytxfee default constexpr CAmount DEFAULT_PAY_TX_FEE = 0; //! -fallbackfee default @@ -99,58 +97,12 @@ struct FeeCalculation; enum class FeeEstimateMode; class ReserveDestination; -/** (client) version numbers for particular wallet features */ -enum WalletFeature -{ - FEATURE_BASE = 10500, // the earliest version new wallets supports (only useful for getwalletinfo's clientversion output) - - FEATURE_WALLETCRYPT = 40000, // wallet encryption - FEATURE_COMPRPUBKEY = 60000, // compressed public keys - - FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) - - FEATURE_HD_SPLIT = 139900, // Wallet with HD chain split (change outputs will use m/0'/1'/k) - - FEATURE_NO_DEFAULT_KEY = 159900, // Wallet without a default key written - - FEATURE_PRE_SPLIT_KEYPOOL = 169900, // Upgraded to HD SPLIT and can have a pre-split keypool - - FEATURE_LATEST = FEATURE_PRE_SPLIT_KEYPOOL -}; - //! Default for -addresstype constexpr OutputType DEFAULT_ADDRESS_TYPE{OutputType::BECH32}; //! Default for -changetype constexpr OutputType DEFAULT_CHANGE_TYPE{OutputType::CHANGE_AUTO}; -enum WalletFlags : uint64_t { - // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown - // unknown wallet flags in the lower section <= (1 << 31) will be tolerated - - // will categorize coins as clean (not reused) and dirty (reused), and handle - // them with privacy considerations in mind - WALLET_FLAG_AVOID_REUSE = (1ULL << 0), - - // Indicates that the metadata has already been upgraded to contain key origins - WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), - - // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) - WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), - - //! Flag set when a wallet contains no HD seed and no private keys, scripts, - //! addresses, and other watch only things, and is therefore "blank." - //! - //! The only function this flag serves is to distinguish a blank wallet from - //! a newly created wallet when the wallet database is loaded, to avoid - //! initialization that should only happen on first run. - //! - //! This flag is also a mandatory flag to prevent previous versions of - //! bitcoin from opening the wallet, thinking it was newly created, and - //! then improperly reinitializing it. - WALLET_FLAG_BLANK_WALLET = (1ULL << 33), -}; - static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET @@ -169,99 +121,6 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ extern const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS; -/** A key from a CWallet's keypool - * - * The wallet holds one (for pre HD-split wallets) or several keypools. These - * are sets of keys that have not yet been used to provide addresses or receive - * change. - * - * The Bitcoin Core wallet was originally a collection of unrelated private - * keys with their associated addresses. If a non-HD wallet generated a - * key/address, gave that address out and then restored a backup from before - * that key's generation, then any funds sent to that address would be - * lost definitively. - * - * The keypool was implemented to avoid this scenario (commit: 10384941). The - * wallet would generate a set of keys (100 by default). When a new public key - * was required, either to give out as an address or to use in a change output, - * it would be drawn from the keypool. The keypool would then be topped up to - * maintain 100 keys. This ensured that as long as the wallet hadn't used more - * than 100 keys since the previous backup, all funds would be safe, since a - * restored wallet would be able to scan for all owned addresses. - * - * A keypool also allowed encrypted wallets to give out addresses without - * having to be decrypted to generate a new private key. - * - * With the introduction of HD wallets (commit: f1902510), the keypool - * essentially became an address look-ahead pool. Restoring old backups can no - * longer definitively lose funds as long as the addresses used were from the - * wallet's HD seed (since all private keys can be rederived from the seed). - * However, if many addresses were used since the backup, then the wallet may - * not know how far ahead in the HD chain to look for its addresses. The - * keypool is used to implement a 'gap limit'. The keypool maintains a set of - * keys (by default 1000) ahead of the last used key and scans for the - * addresses of those keys. This avoids the risk of not seeing transactions - * involving the wallet's addresses, or of re-using the same address. - * - * The HD-split wallet feature added a second keypool (commit: 02592f4c). There - * is an external keypool (for addresses to hand out) and an internal keypool - * (for change addresses). - * - * Keypool keys are stored in the wallet/keystore's keymap. The keypool data is - * stored as sets of indexes in the wallet (setInternalKeyPool, - * setExternalKeyPool and set_pre_split_keypool), and a map from the key to the - * index (m_pool_key_to_index). The CKeyPool object is used to - * serialize/deserialize the pool data to/from the database. - */ -class CKeyPool -{ -public: - //! The time at which the key was generated. Set in AddKeypoolPubKeyWithDB - int64_t nTime; - //! The public key - CPubKey vchPubKey; - //! Whether this keypool entry is in the internal keypool (for change outputs) - bool fInternal; - //! Whether this key was generated for a keypool before the wallet was upgraded to HD-split - bool m_pre_split; - - CKeyPool(); - CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(fInternal); - } - catch (std::ios_base::failure&) { - /* flag as external address if we can't read the internal boolean - (this will be the case for any wallet before the HD chain split version) */ - fInternal = false; - } - try { - READWRITE(m_pre_split); - } - catch (std::ios_base::failure&) { - /* flag as postsplit address if we can't read the m_pre_split boolean - (this will be the case for any wallet that upgrades to HD chain split)*/ - m_pre_split = false; - } - } - else { - READWRITE(fInternal); - READWRITE(m_pre_split); - } - } -}; - /** A wrapper to reserve an address from a wallet * * ReserveDestination is used to reserve an address. @@ -282,6 +141,7 @@ class ReserveDestination protected: //! The wallet to reserve from CWallet* pwallet; + LegacyScriptPubKeyMan* m_spk_man{nullptr}; //! The index of the address's key in the keypool int64_t nIndex{-1}; //! The public key for the address @@ -496,14 +356,17 @@ public: ABANDONED }; - /* Confirmation includes tx status and a pair of {block hash/tx index in block} at which tx has been confirmed. - * This pair is both 0 if tx hasn't confirmed yet. Meaning of these fields changes with CONFLICTED state - * where they instead point to block hash and index of the deepest conflicting tx. + /* Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} + * at which tx has been confirmed. All three are set to 0 if tx is unconfirmed or abandoned. + * Meaning of these fields changes with CONFLICTED state where they instead point to block hash + * and block height of the deepest conflicting tx. */ struct Confirmation { - Status status = UNCONFIRMED; - uint256 hashBlock = uint256(); - int nIndex = 0; + Status status; + int block_height; + uint256 hashBlock; + int nIndex; + Confirmation(Status s = UNCONFIRMED, int b = 0, uint256 h = uint256(), int i = 0) : status(s), block_height(b), hashBlock(h), nIndex(i) {} }; Confirmation m_confirm; @@ -546,7 +409,6 @@ public: * compatibility (pre-commit 9ac63d6). */ if (serializedIndex == -1 && m_confirm.hashBlock == ABANDON_HASH) { - m_confirm.hashBlock = uint256(); setAbandoned(); } else if (serializedIndex == -1) { setConflicted(); @@ -587,14 +449,14 @@ public: //! filter decides which addresses will count towards the debit CAmount GetDebit(const isminefilter& filter) const; - CAmount GetCredit(interfaces::Chain::Lock& locked_chain, const isminefilter& filter) const; - CAmount GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true) const; + CAmount GetCredit(const isminefilter& filter) const; + CAmount GetImmatureCredit(bool fUseCache = true) const; // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid // having to resolve the issue of member access into incomplete type CWallet. - CAmount GetAvailableCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache=true, const isminefilter& filter=ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; - CAmount GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache=true) const; + CAmount GetAvailableCredit(bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; + CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const; CAmount GetChange() const; // Get the marginal bytes if spending the specified output from this transaction @@ -616,11 +478,12 @@ public: bool InMempool() const; bool IsTrusted(interfaces::Chain::Lock& locked_chain) const; + bool IsTrusted(interfaces::Chain::Lock& locked_chain, std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; // Pass this transaction to node for mempool insertion and relay to peers if flag set to true - bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay, interfaces::Chain::Lock& locked_chain); + bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation @@ -630,38 +493,44 @@ public: // in place. std::set<uint256> GetConflicts() const NO_THREAD_SAFETY_ANALYSIS; - void SetConf(Status status, const uint256& block_hash, int posInBlock); - /** * Return depth of transaction in blockchain: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain */ - int GetDepthInMainChain(interfaces::Chain::Lock& locked_chain) const; - bool IsInMainChain(interfaces::Chain::Lock& locked_chain) const { return GetDepthInMainChain(locked_chain) > 0; } + // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation + // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to + // resolve the issue of member access into incomplete type CWallet. Note + // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" + // in place. + int GetDepthInMainChain() const NO_THREAD_SAFETY_ANALYSIS; + bool IsInMainChain() const { return GetDepthInMainChain() > 0; } /** * @return number of blocks to maturity for this transaction: * 0 : is not a coinbase transaction, or is a mature coinbase transaction * >0 : is a coinbase transaction which matures in this many blocks */ - int GetBlocksToMaturity(interfaces::Chain::Lock& locked_chain) const; + int GetBlocksToMaturity() const; bool isAbandoned() const { return m_confirm.status == CWalletTx::ABANDONED; } void setAbandoned() { m_confirm.status = CWalletTx::ABANDONED; m_confirm.hashBlock = uint256(); + m_confirm.block_height = 0; m_confirm.nIndex = 0; } bool isConflicted() const { return m_confirm.status == CWalletTx::CONFLICTED; } void setConflicted() { m_confirm.status = CWalletTx::CONFLICTED; } bool isUnconfirmed() const { return m_confirm.status == CWalletTx::UNCONFIRMED; } void setUnconfirmed() { m_confirm.status = CWalletTx::UNCONFIRMED; } + bool isConfirmed() const { return m_confirm.status == CWalletTx::CONFIRMED; } void setConfirmed() { m_confirm.status = CWalletTx::CONFIRMED; } const uint256& GetHash() const { return tx->GetHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } - bool IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const; + bool IsImmatureCoinBase() const; }; class COutput @@ -722,10 +591,9 @@ struct CoinSelectionParams class WalletRescanReserver; //forward declarations for ScanForWalletTransactions/RescanFromTime /** - * A CWallet is an extension of a keystore, which also maintains a set of transactions and balances, - * and provides the ability to create new transactions. + * A CWallet maintains a set of transactions and balances, and provides the ability to create new transactions. */ -class CWallet final : public FillableSigningProvider, private interfaces::Chain::Notifications +class CWallet final : public WalletStorage, private interfaces::Chain::Notifications { private: CKeyingMaterial vMasterKey GUARDED_BY(cs_KeyStore); @@ -737,22 +605,8 @@ private: //! keeps track of whether Unlock has run a thorough check before bool fDecryptionThoroughlyChecked; - using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; - using WatchOnlySet = std::set<CScript>; - using WatchKeyMap = std::map<CKeyID, CPubKey>; - bool SetCrypted(); - - //! will encrypt previously unencrypted keys - bool EncryptKeys(CKeyingMaterial& vMasterKeyIn); - bool Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys = false); - CryptedKeyMap mapCryptedKeys GUARDED_BY(cs_KeyStore); - WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); - WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); - - bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); std::atomic<bool> fAbortRescan{false}; std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver @@ -761,8 +615,6 @@ private: std::mutex mutexScanning; friend class WalletRescanReserver; - WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; - //! the current wallet version: clients below this version are not able to load the wallet int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; @@ -798,10 +650,10 @@ private: * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. */ - bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Status status, const uint256& block_hash, int posInBlock, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /* Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ - void MarkConflicted(const uint256& hashBlock, const uint256& hashTx); + void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); /* Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -810,55 +662,18 @@ private: /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ - void SyncTransaction(const CTransactionRef& tx, CWalletTx::Status status, const uint256& block_hash, int posInBlock = 0, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - /* the HD chain data model (external chain counters) */ - CHDChain hdChain; - - /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); - int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; - std::map<CKeyID, int64_t> m_pool_key_to_index; std::atomic<uint64_t> m_wallet_flags{0}; - int64_t nTimeFirstKey GUARDED_BY(cs_wallet) = 0; - - /** - * Private version of AddWatchOnly method which does not accept a - * timestamp, and which will reset the wallet's nTimeFirstKey value to 1 if - * the watch key did not previously have a timestamp associated with it. - * Because this is an inherited virtual method, it is accessible despite - * being marked private, but it is marked private anyway to encourage use - * of the other AddWatchOnly which accepts a timestamp and sets - * nTimeFirstKey more intelligently for more efficient rescans. - */ - bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnlyInMem(const CScript &dest); - - /** Add a KeyOriginInfo to the wallet */ - bool AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info); - - //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); - bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose); - //! Adds a script to the store and saves it to disk - bool AddCScriptWithDB(WalletBatch& batch, const CScript& script); - //! Unsets a wallet flag and saves it to disk void UnsetWalletFlagWithDB(WalletBatch& batch, uint64_t flag); + //! Unset the blank wallet flag and saves it to disk + void UnsetBlankWalletFlag(WalletBatch& batch) override; + /** Interface for accessing chain state. */ interfaces::Chain* m_chain; @@ -872,14 +687,17 @@ private: * The following is used to keep track of how far behind the wallet is * from the chain sync, and to allow clients to block on us being caught up. * - * Note that this is *not* how far we've processed, we may need some rescan - * to have seen all transactions in the chain, but is only used to track - * live BlockConnected callbacks. + * Processed hash is a pointer on node's tip and doesn't imply that the wallet + * has scanned sequentially all blocks up to this one. */ uint256 m_last_block_processed GUARDED_BY(cs_wallet); - //! Fetches a key from the keypool - bool GetKeyFromPool(CPubKey &key, bool internal = false); + /* Height of last block processed is used by wallet to know depth of transactions + * without relying on Chain interface beyond asynchronous updates. For safety, we + * initialize it to -1. Height is a pointer on node's tip and doesn't imply + * that the wallet has scanned sequentially all blocks up to this one. + */ + int m_last_block_processed_height GUARDED_BY(cs_wallet) = -1; public: /* @@ -895,6 +713,7 @@ public: { return *database; } + WalletDatabase& GetDatabase() override { return *database; } /** * Select a set of coins such that nValueRet >= nTargetValue and at least @@ -910,15 +729,6 @@ public: */ const std::string& GetName() const { return m_location.GetName(); } - void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - // Map from Key ID to key metadata. - std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_wallet); - - // Map from Script ID to key metadata (for watch-only keys). - std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); - typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; @@ -942,7 +752,7 @@ public: } bool IsCrypted() const { return fUseCrypto; } - bool IsLocked() const; + bool IsLocked() const override; bool Lock(); /** Interface to assert chain access and if successful lock it */ @@ -972,7 +782,7 @@ public: const CWalletTx* GetWalletTx(const uint256& hash) const; //! check whether we are allowed to upgrade (or already support) to the named feature - bool CanSupportFeature(enum WalletFeature wf) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } + bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } /** * populate vCoins with vector of available COutputs. @@ -998,12 +808,12 @@ public: bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; - bool IsSpent(interfaces::Chain::Lock& locked_chain, const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); // Whether this or any UTXO with the same CTxDestination has been spent. bool IsUsedDestination(const CTxDestination& dst) const; bool IsUsedDestination(const uint256& hash, unsigned int n) const; - void SetUsedDestinationState(const uint256& hash, unsigned int n, bool used); + void SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; @@ -1022,39 +832,15 @@ public: int64_t ScanningDuration() const { return fScanningWallet ? GetTimeMillis() - m_scanning_start : 0; } double ScanningProgress() const { return fScanningWallet ? (double) m_scanning_progress : 0; } - /** - * keystore implementation - * Generate a new key - */ - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a key to the store, without saving it to disk (used by LoadWallet) - bool LoadKey(const CKey& key, const CPubKey &pubkey) { return AddKeyPubKeyInner(key, pubkey); } - //! Load metadata (used by LoadWallet) - void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } - void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - //! Adds an encrypted key to the store, and saves it to disk. - bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) - bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - bool GetKey(const CKeyID &address, CKey& keyOut) const override; - bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - bool HaveKey(const CKeyID &address) const override; - std::set<CKeyID> GetKeys() const override; - bool AddCScript(const CScript& redeemScript) override; - bool LoadCScript(const CScript& redeemScript); //! Adds a destination data tuple to the store, and saves it to disk - bool AddDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Erases a destination data tuple in the store and on disk - bool EraseDestData(const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool EraseDestData(WalletBatch& batch, const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a destination data tuple to the store, without saving it to disk void LoadDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Look up a destination data tuple in the store, return true if found false otherwise @@ -1062,18 +848,6 @@ public: //! Get all destination values matching a prefix. std::vector<std::string> GetDestValues(const std::string& prefix) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - //! Adds a watch-only address to the store, without saving it to disk (used by LoadWallet) - bool LoadWatchOnly(const CScript &dest); - //! Returns whether the watch-only script is in the wallet - bool HaveWatchOnly(const CScript &dest) const; - //! Returns whether there are any watch-only things in the wallet - bool HaveWatchOnly() const; - //! Fetches a pubkey from mapWatchKeys if it exists there - bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; - //! Holds a timestamp at which point the wallet is scheduled (externally) to be relocked. Caller must arrange for actual relocking to occur via Lock(). int64_t nRelockTime = 0; @@ -1095,8 +869,8 @@ public: bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); void LoadToWallet(CWalletTx& wtxIn) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void TransactionAddedToMempool(const CTransactionRef& tx) override; - void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const CBlock& block) override; + void BlockConnected(const CBlock& block, const std::vector<CTransactionRef>& vtxConflicted, int height) override; + void BlockDisconnected(const CBlock& block, int height) override; void UpdatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); @@ -1117,7 +891,7 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; - void ReacceptWalletTransactions(interfaces::Chain::Lock& locked_chain) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); struct Balance { CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more @@ -1146,7 +920,16 @@ public: */ bool CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign = true); - bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CValidationState& state); + /** + * Submit the transaction to the node's mempool and then relay to peers. + * Should be called after CreateTransaction unless you want to abort + * broadcasting the transaction. + * + * @param tx[in] The transaction to be broadcast. + * @param mapValue[in] key-values to be set on the transaction. + * @param orderForm[in] BIP 70 / BIP 21 order form details to be set on the transaction. + */ + void CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm); bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const { @@ -1180,33 +963,10 @@ public: /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; - bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool TopUpKeyPool(unsigned int kpSize = 0); - /** - * Reserves a key from the keypool and sets nIndex to its index - * - * @param[out] nIndex the index of the key in keypool - * @param[out] keypool the keypool the key was drawn from, which could be the - * the pre-split pool if present, or the internal or external pool - * @param fRequestedInternal true if the caller would like the key drawn - * from the internal keypool, false if external is preferred - * - * @return true if succeeded, false if failed due to empty keypool - * @throws std::runtime_error if keypool read failed, key was invalid, - * was not found in the wallet, or was misclassified in the internal - * or external keypool - */ - bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); - void KeepKey(int64_t nIndex); - void ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey); int64_t GetOldestKeyPoolTime(); - /** - * Marks all keys in the keypool up to and including reserve_key as used. - */ - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set<std::set<CTxDestination>> GetAddressGroupings() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::map<CTxDestination, CAmount> GetAddressBalances(interfaces::Chain::Lock& locked_chain); @@ -1216,6 +976,8 @@ public: bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error); + isminetype IsMine(const CTxDestination& dest) const; + isminetype IsMine(const CScript& script) const; isminetype IsMine(const CTxIn& txin) const; /** * Returns amount of debit if the input matches the @@ -1245,14 +1007,10 @@ public: bool DelAddressBook(const CTxDestination& address); - unsigned int GetKeyPoolSize() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) - { - AssertLockHeld(cs_wallet); - return setInternalKeyPool.size() + setExternalKeyPool.size(); - } + unsigned int GetKeyPoolSize() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! signify that a particular wallet feature is now used. this may change nWalletVersion and nWalletMaxVersion if those are lower - void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false); + void SetMinVersion(enum WalletFeature, WalletBatch* batch_in = nullptr, bool fExplicit = false) override; //! change which version we're allowed to upgrade to (note that this does not immediately imply upgrading to that format) bool SetMaxVersion(int nVersion); @@ -1312,7 +1070,7 @@ public: bool TransactionCanBeAbandoned(const uint256& hashTx) const; /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ - bool AbandonTransaction(interfaces::Chain::Lock& locked_chain, const uint256& hashTx); + bool AbandonTransaction(const uint256& hashTx); /** Mark a transaction as replaced by another transaction (e.g., BIP 125). */ bool MarkReplaced(const uint256& originalHash, const uint256& newHash); @@ -1331,31 +1089,12 @@ public: bool BackupWallet(const std::string& strDest); - /* Set the HD chain model (chain child index counters) */ - void SetHDChain(const CHDChain& chain, bool memonly); - const CHDChain& GetHDChain() const { return hdChain; } - /* Returns true if HD is enabled */ bool IsHDEnabled() const; - /* Returns true if the wallet can generate new keys */ - bool CanGenerateKeys(); - /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ bool CanGetAddresses(bool internal = false); - /* Generates a new HD seed (will not be activated) */ - CPubKey GenerateNewSeed(); - - /* Derives a new HD seed (will not be activated) */ - CPubKey DeriveNewSeed(const CKey& key); - - /* Set the current HD seed (will reset the chain child index counters) - Sets the seed's version based on the current wallet version (so the - caller must ensure the current wallet version is correct before calling - this function). */ - void SetHDSeed(const CPubKey& key); - /** * Blocks until the wallet state is up-to-date to /at least/ the current * chain at the time this function is entered @@ -1364,20 +1103,6 @@ public: */ void BlockUntilSyncedToCurrentChain() LOCKS_EXCLUDED(cs_main, cs_wallet); - /** - * Explicitly make the wallet learn the related scripts for outputs to the - * given key. This is purely to make the wallet file compatible with older - * software, as FillableSigningProvider automatically does this implicitly for all - * keys now. - */ - void LearnRelatedScripts(const CPubKey& key, OutputType); - - /** - * Same as LearnRelatedScripts, but when the OutputType is not known (and could - * be anything). - */ - void LearnAllRelatedScripts(const CPubKey& key); - /** set a single wallet flag */ void SetWalletFlag(uint64_t flags); @@ -1385,14 +1110,14 @@ public: void UnsetWalletFlag(uint64_t flag); /** check if a certain wallet flag is set */ - bool IsWalletFlagSet(uint64_t flag) const; + bool IsWalletFlagSet(uint64_t flag) const override; /** overwrite all flags by the given uint64_t returns false if unknown, non-tolerable flags are present */ bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); /** Returns a bracketed wallet name for displaying in logs, will return [default wallet] if the wallet has no name */ - const std::string GetDisplayName() const { + const std::string GetDisplayName() const override { std::string wallet_name = GetName().length() == 0 ? "default wallet" : GetName(); return strprintf("[%s]", wallet_name); }; @@ -1403,8 +1128,36 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; - /** Implement lookup of key origin information through wallet key metadata. */ - bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + ScriptPubKeyMan* GetScriptPubKeyMan() const; + const SigningProvider* GetSigningProvider() const; + LegacyScriptPubKeyMan* GetLegacyScriptPubKeyMan() const; + + // Temporary LegacyScriptPubKeyMan accessors and aliases. + friend class LegacyScriptPubKeyMan; + std::unique_ptr<LegacyScriptPubKeyMan> m_spk_man = MakeUnique<LegacyScriptPubKeyMan>(*this); + CCriticalSection& cs_KeyStore = m_spk_man->cs_KeyStore; + LegacyScriptPubKeyMan::KeyMap& mapKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapKeys; + LegacyScriptPubKeyMan::ScriptMap& mapScripts GUARDED_BY(cs_KeyStore) = m_spk_man->mapScripts; + LegacyScriptPubKeyMan::CryptedKeyMap& mapCryptedKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapCryptedKeys; + LegacyScriptPubKeyMan::WatchOnlySet& setWatchOnly GUARDED_BY(cs_KeyStore) = m_spk_man->setWatchOnly; + LegacyScriptPubKeyMan::WatchKeyMap& mapWatchKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapWatchKeys; + WalletBatch*& encrypted_batch GUARDED_BY(cs_wallet) = m_spk_man->encrypted_batch; + using CryptedKeyMap = LegacyScriptPubKeyMan::CryptedKeyMap; + + /** Get last block processed height */ + int GetLastBlockHeight() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + assert(m_last_block_processed_height >= 0); + return m_last_block_processed_height; + }; + /** Set last block processed height, currently only use in unit test */ + void SetLastBlockProcessed(int block_height, uint256 block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + m_last_block_processed_height = block_height; + m_last_block_processed = block_hash; + }; }; /** diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index d7f7635a83..7d04b04764 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -5,8 +5,6 @@ #include <wallet/walletdb.h> -#include <consensus/tx_check.h> -#include <consensus/validation.h> #include <fs.h> #include <key_io.h> #include <protocol.h> @@ -198,7 +196,7 @@ public: static bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, - CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) + CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet, pwallet->GetLegacyScriptPubKeyMan()->cs_wallet) { try { // Unserialize @@ -218,8 +216,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssKey >> hash; CWalletTx wtx(nullptr /* pwallet */, MakeTransactionRef()); ssValue >> wtx; - CValidationState state; - if (!(CheckTransaction(*wtx.tx, state) && (wtx.GetHash() == hash) && state.IsValid())) + if (wtx.GetHash() != hash) return false; // Undo serialize changes in 31600 @@ -253,8 +250,9 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssKey >> script; char fYes; ssValue >> fYes; - if (fYes == '1') - pwallet->LoadWatchOnly(script); + if (fYes == '1') { + pwallet->GetLegacyScriptPubKeyMan()->LoadWatchOnly(script); + } } else if (strType == DBKeys::KEY) { CPubKey vchPubKey; ssKey >> vchPubKey; @@ -305,12 +303,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strErr = "Error reading wallet database: CPrivKey corrupt"; return false; } - if (!pwallet->LoadKey(key, vchPubKey)) + if (!pwallet->GetLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) { - strErr = "Error reading wallet database: LoadKey failed"; + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadKey failed"; return false; } } else if (strType == DBKeys::MASTER_KEY) { + // Master encryption key is loaded into only the wallet and not any of the ScriptPubKeyMans. unsigned int nID; ssKey >> nID; CMasterKey kMasterKey; @@ -335,9 +334,9 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> vchPrivKey; wss.nCKeys++; - if (!pwallet->LoadCryptedKey(vchPubKey, vchPrivKey)) + if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) { - strErr = "Error reading wallet database: LoadCryptedKey failed"; + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; } wss.fIsEncrypted = true; @@ -347,14 +346,14 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + pwallet->GetLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->LoadScriptMetadata(CScriptID(script), keyMeta); + pwallet->GetLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); } else if (strType == DBKeys::DEFAULTKEY) { // We don't want or need the default key, but if there is one set, // we want to make sure that it is valid so that we can detect corruption @@ -370,15 +369,15 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyPool keypool; ssValue >> keypool; - pwallet->LoadKeyPool(nIndex, keypool); + pwallet->GetLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); } else if (strType == DBKeys::CSCRIPT) { uint160 hash; ssKey >> hash; CScript script; ssValue >> script; - if (!pwallet->LoadCScript(script)) + if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCScript(script)) { - strErr = "Error reading wallet database: LoadCScript failed"; + strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCScript failed"; return false; } } else if (strType == DBKeys::ORDERPOSNEXT) { @@ -392,7 +391,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; - pwallet->SetHDChain(chain, true); + pwallet->GetLegacyScriptPubKeyMan()->SetHDChain(chain, true); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; @@ -435,6 +434,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) DBErrors result = DBErrors::LOAD_OK; LOCK(pwallet->cs_wallet); + AssertLockHeld(pwallet->GetLegacyScriptPubKeyMan()->cs_wallet); try { int nMinVersion = 0; if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { @@ -515,8 +515,12 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); // nTimeFirstKey is only reliable if all keys have metadata - if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) - pwallet->UpdateTimeFirstKey(1); + if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { + auto spk_man = pwallet->GetLegacyScriptPubKeyMan(); + if (spk_man) { + spk_man->UpdateTimeFirstKey(1); + } + } for (const uint256& hash : wss.vWalletUpgrade) WriteTx(pwallet->mapWallet.at(hash)); @@ -625,7 +629,7 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u } else if ((*it) == hash) { if(!EraseTx(hash)) { - LogPrint(BCLog::DB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); + LogPrint(BCLog::WALLETDB, "Transaction was found for deletion but returned database error: %s\n", hash.GetHex()); delerror = true; } vTxHashOut.push_back(hash); @@ -709,6 +713,7 @@ bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, C { // Required in LoadKeyMetadata(): LOCK(dummyWallet->cs_wallet); + AssertLockHeld(dummyWallet->GetLegacyScriptPubKeyMan()->cs_wallet); fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, dummyWss, strType, strErr); } diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 0843194511..dc0cac60bd 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -30,15 +30,16 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: bool first_run = true; DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); if (load_wallet_ret != DBErrors::LOAD_OK) { - tfm::format(std::cerr, "Error creating %s", name.c_str()); + tfm::format(std::cerr, "Error creating %s", name); return nullptr; } wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); // generate a new HD seed - CPubKey seed = wallet_instance->GenerateNewSeed(); - wallet_instance->SetHDSeed(seed); + auto spk_man = wallet_instance->GetLegacyScriptPubKeyMan(); + CPubKey seed = spk_man->GenerateNewSeed(); + spk_man->SetHDSeed(seed); tfm::format(std::cout, "Topping up keypool...\n"); wallet_instance->TopUpKeyPool(); @@ -59,28 +60,28 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa bool first_run; load_wallet_ret = wallet_instance->LoadWallet(first_run); } catch (const std::runtime_error&) { - tfm::format(std::cerr, "Error loading %s. Is wallet being used by another process?\n", name.c_str()); + tfm::format(std::cerr, "Error loading %s. Is wallet being used by another process?\n", name); return nullptr; } if (load_wallet_ret != DBErrors::LOAD_OK) { wallet_instance = nullptr; if (load_wallet_ret == DBErrors::CORRUPT) { - tfm::format(std::cerr, "Error loading %s: Wallet corrupted", name.c_str()); + tfm::format(std::cerr, "Error loading %s: Wallet corrupted", name); return nullptr; } else if (load_wallet_ret == DBErrors::NONCRITICAL_ERROR) { tfm::format(std::cerr, "Error reading %s! All keys read correctly, but transaction data" " or address book entries might be missing or incorrect.", - name.c_str()); + name); } else if (load_wallet_ret == DBErrors::TOO_NEW) { tfm::format(std::cerr, "Error loading %s: Wallet requires newer version of %s", - name.c_str(), PACKAGE_NAME); + name, PACKAGE_NAME); return nullptr; } else if (load_wallet_ret == DBErrors::NEED_REWRITE) { tfm::format(std::cerr, "Wallet needed to be rewritten: restart %s to complete", PACKAGE_NAME); return nullptr; } else { - tfm::format(std::cerr, "Error loading %s", name.c_str()); + tfm::format(std::cerr, "Error loading %s", name); return nullptr; } } @@ -94,7 +95,7 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Wallet info\n===========\n"); tfm::format(std::cout, "Encrypted: %s\n", wallet_instance->IsCrypted() ? "yes" : "no"); - tfm::format(std::cout, "HD (hd seed available): %s\n", wallet_instance->GetHDChain().seed_id.IsNull() ? "no" : "yes"); + tfm::format(std::cout, "HD (hd seed available): %s\n", wallet_instance->IsHDEnabled() ? "yes" : "no"); tfm::format(std::cout, "Keypool Size: %u\n", wallet_instance->GetKeyPoolSize()); tfm::format(std::cout, "Transactions: %zu\n", wallet_instance->mapWallet.size()); tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->mapAddressBook.size()); @@ -112,12 +113,12 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) } } else if (command == "info") { if (!fs::exists(path)) { - tfm::format(std::cerr, "Error: no wallet file at %s\n", name.c_str()); + tfm::format(std::cerr, "Error: no wallet file at %s\n", name); return false; } std::string error; if (!WalletBatch::VerifyEnvironment(path, error)) { - tfm::format(std::cerr, "Error loading %s. Is wallet being used by other process?\n", name.c_str()); + tfm::format(std::cerr, "Error loading %s. Is wallet being used by other process?\n", name); return false; } std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); @@ -125,7 +126,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) WalletShowInfo(wallet_instance.get()); wallet_instance->Flush(true); } else { - tfm::format(std::cerr, "Invalid command: %s\n", command.c_str()); + tfm::format(std::cerr, "Invalid command: %s\n", command); return false; } diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index ba2f913841..044c757e68 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -9,6 +9,54 @@ #include <vector> +/** (client) version numbers for particular wallet features */ +enum WalletFeature +{ + FEATURE_BASE = 10500, // the earliest version new wallets supports (only useful for getwalletinfo's clientversion output) + + FEATURE_WALLETCRYPT = 40000, // wallet encryption + FEATURE_COMPRPUBKEY = 60000, // compressed public keys + + FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) + + FEATURE_HD_SPLIT = 139900, // Wallet with HD chain split (change outputs will use m/0'/1'/k) + + FEATURE_NO_DEFAULT_KEY = 159900, // Wallet without a default key written + + FEATURE_PRE_SPLIT_KEYPOOL = 169900, // Upgraded to HD SPLIT and can have a pre-split keypool + + FEATURE_LATEST = FEATURE_PRE_SPLIT_KEYPOOL +}; + + + +enum WalletFlags : uint64_t { + // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown + // unknown wallet flags in the lower section <= (1 << 31) will be tolerated + + // will categorize coins as clean (not reused) and dirty (reused), and handle + // them with privacy considerations in mind + WALLET_FLAG_AVOID_REUSE = (1ULL << 0), + + // Indicates that the metadata has already been upgraded to contain key origins + WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), + + // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) + WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), + + //! Flag set when a wallet contains no HD seed and no private keys, scripts, + //! addresses, and other watch only things, and is therefore "blank." + //! + //! The only function this flag serves is to distinguish a blank wallet from + //! a newly created wallet when the wallet database is loaded, to avoid + //! initialization that should only happen on first run. + //! + //! This flag is also a mandatory flag to prevent previous versions of + //! bitcoin from opening the wallet, thinking it was newly created, and + //! then improperly reinitializing it. + WALLET_FLAG_BLANK_WALLET = (1ULL << 33), +}; + //! Get the path of the wallet directory. fs::path GetWalletDir(); diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h index 2e1fdf4f3a..7ccda1c566 100644 --- a/src/walletinitinterface.h +++ b/src/walletinitinterface.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLETINITINTERFACE_H #define BITCOIN_WALLETINITINTERFACE_H -struct InitInterfaces; +struct NodeContext; class WalletInitInterface { public: @@ -15,8 +15,8 @@ public: virtual void AddWalletOptions() const = 0; /** Check wallet parameter interaction */ virtual bool ParameterInteraction() const = 0; - /** Add wallets that should be opened to list of init interfaces. */ - virtual void Construct(InitInterfaces& interfaces) const = 0; + /** Add wallets that should be opened to list of chain clients. */ + virtual void Construct(NodeContext& node) const = 0; virtual ~WalletInitInterface() {} }; diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index ebbaf8683d..b3b97b6a2a 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -185,7 +185,7 @@ void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBloc } } -void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) +void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) { for (const CTransactionRef& ptx : pblock->vtx) { // Do a normal notify for each transaction removed in block disconnection diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 6be0554a65..8bf9b0ba47 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -27,7 +27,7 @@ protected: // CValidationInterface void TransactionAddedToMempool(const CTransactionRef& tx) override; void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; - void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; + void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; private: diff --git a/test/README.md b/test/README.md index 11adc11278..24a9389fac 100644 --- a/test/README.md +++ b/test/README.md @@ -254,7 +254,13 @@ Use the `-v` option for verbose output. #### Dependencies -The lint tests require codespell and flake8. To install: `pip3 install codespell flake8`. +| Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation +|-----------|:----------:|:-------------------------------------------:|-------------- +| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.7.8](https://github.com/bitcoin/bitcoin/pull/15257) | `pip3 install flake8==3.7.8` +| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.6.0](https://github.com/bitcoin/bitcoin/pull/15166) | [details...](https://github.com/koalaman/shellcheck#installing) +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [1.15.0](https://github.com/bitcoin/bitcoin/pull/16186) | `pip3 install codespell==1.15.0` + +Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. #### Running the tests diff --git a/test/functional/README.md b/test/functional/README.md index a9b83076eb..77a9ce9acb 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -99,6 +99,16 @@ P2PInterface object and override the callback methods. Examples tests are [p2p_unrequested_blocks.py](p2p_unrequested_blocks.py), [p2p_compactblocks.py](p2p_compactblocks.py). +#### Prototyping tests + +The [`TestShell`](test-shell.md) class exposes the BitcoinTestFramework +functionality to interactive Python3 environments and can be used to prototype +tests. This may be especially useful in a REPL environment with session logging +utilities, such as +[IPython](https://ipython.readthedocs.io/en/stable/interactive/reference.html#session-logging-and-restoring). +The logs of such interactive sessions can later be adapted into permanent test +cases. + ### Test framework modules The following are useful modules for test developers. They are located in [test/functional/test_framework/](test_framework). diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 682a00ff4d..fe45ed34b5 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -24,7 +24,7 @@ SEQUENCE_LOCKTIME_GRANULARITY = 9 # this is a bit-shift SEQUENCE_LOCKTIME_MASK = 0x0000ffff # RPC error for non-BIP68 final transactions -NOT_FINAL_ERROR = "non-BIP68-final (code 64)" +NOT_FINAL_ERROR = "non-BIP68-final" class BIP68Test(BitcoinTestFramework): def set_test_params(self): diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 13f1c9216a..7d131e6045 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -126,7 +126,7 @@ class BIP65Test(BitcoinTestFramework): # First we show that this tx is valid except for CLTV by getting it # rejected from the mempool for exactly that reason. assert_equal( - [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Negative locktime)'}], + [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': 'non-mandatory-script-verify-flag (Negative locktime)'}], self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 92f6eea5b2..2ace96fef4 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -110,7 +110,7 @@ class BIP66Test(BitcoinTestFramework): # First we show that this tx is valid except for DERSIG by getting it # rejected from the mempool for exactly that reason. assert_equal( - [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Non-canonical DER signature)'}], + [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)'}], self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0) ) diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py new file mode 100755 index 0000000000..bf2a4ff61f --- /dev/null +++ b/test/functional/feature_loadblock.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test loadblock option + +Test the option to start a node with the option loadblock which loads +a serialized blockchain from a file (usually called bootstrap.dat). +To generate that file this test uses the helper scripts available +in contrib/linearize. +""" + +import os +import subprocess +import sys +import tempfile +import urllib + +from test_framework.test_framework import ( + BitcoinTestFramework, +) +from test_framework.util import assert_equal, wait_until + + +class LoadblockTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + + def run_test(self): + self.nodes[1].setnetworkactive(state=False) + self.nodes[0].generate(100) + + # Parsing the url of our node to get settings for config file + data_dir = self.nodes[0].datadir + node_url = urllib.parse.urlparse(self.nodes[0].url) + cfg_file = os.path.join(data_dir, "linearize.cfg") + bootstrap_file = os.path.join(self.options.tmpdir, "bootstrap.dat") + genesis_block = self.nodes[0].getblockhash(0) + blocks_dir = os.path.join(data_dir, "regtest", "blocks") + hash_list = tempfile.NamedTemporaryFile(dir=data_dir, + mode='w', + delete=False, + encoding="utf-8") + + self.log.info("Create linearization config file") + with open(cfg_file, "a", encoding="utf-8") as cfg: + cfg.write("datadir={}\n".format(data_dir)) + cfg.write("rpcuser={}\n".format(node_url.username)) + cfg.write("rpcpassword={}\n".format(node_url.password)) + cfg.write("port={}\n".format(node_url.port)) + cfg.write("host={}\n".format(node_url.hostname)) + cfg.write("output_file={}\n".format(bootstrap_file)) + cfg.write("max_height=100\n") + cfg.write("netmagic=fabfb5da\n") + cfg.write("input={}\n".format(blocks_dir)) + cfg.write("genesis={}\n".format(genesis_block)) + cfg.write("hashlist={}\n".format(hash_list.name)) + + base_dir = self.config["environment"]["SRCDIR"] + linearize_dir = os.path.join(base_dir, "contrib", "linearize") + + self.log.info("Run linearization of block hashes") + linearize_hashes_file = os.path.join(linearize_dir, "linearize-hashes.py") + subprocess.run([sys.executable, linearize_hashes_file, cfg_file], + stdout=hash_list, + check=True) + + self.log.info("Run linearization of block data") + linearize_data_file = os.path.join(linearize_dir, "linearize-data.py") + subprocess.run([sys.executable, linearize_data_file, cfg_file], + check=True) + + self.log.info("Restart second, unsynced node with bootstrap file") + self.stop_node(1) + self.start_node(1, ["-loadblock=" + bootstrap_file]) + wait_until(lambda: self.nodes[1].getblockcount() == 100) + + assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 100) + assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash()) + + +if __name__ == '__main__': + LoadblockTest().main() diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 250dee1528..aaf56a42d0 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -20,7 +20,7 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error -NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero) (code 64)" +NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" def trueDummy(tx): scriptSig = CScript(tx.vin[0].scriptSig) diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index c69c7f90e8..82c7e55245 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -193,10 +193,10 @@ class SegWitTest(BitcoinTestFramework): assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness) (code 64)', wit_ids[NODE_2][WIT_V1][2], sign=False) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', p2sh_ids[NODE_2][WIT_V0][2], sign=False, redeem_script=witness_script(False, self.pubkey[2])) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness) (code 64)', p2sh_ids[NODE_2][WIT_V1][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', wit_ids[NODE_2][WIT_V0][2], sign=False) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', wit_ids[NODE_2][WIT_V1][2], sign=False) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', p2sh_ids[NODE_2][WIT_V0][2], sign=False, redeem_script=witness_script(False, self.pubkey[2])) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][WIT_V1][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) # block 432 diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index dee7a04516..c466b120f2 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -71,7 +71,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): node.generate(1) self.mempool_size = 0 self.check_mempool_result( - result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': '18: txn-already-known'}], + result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': 'txn-already-known'}], rawtxs=[raw_tx_in_block], ) @@ -109,7 +109,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): node.sendrawtransaction(hexstring=raw_tx_0) self.mempool_size += 1 self.check_mempool_result( - result_expected=[{'txid': txid_0, 'allowed': False, 'reject-reason': '18: txn-already-in-mempool'}], + result_expected=[{'txid': txid_0, 'allowed': False, 'reject-reason': 'txn-already-in-mempool'}], rawtxs=[raw_tx_0], ) @@ -133,7 +133,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee # skip re-signing the tx self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '18: txn-mempool-conflict'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'txn-mempool-conflict'}], rawtxs=[tx.serialize().hex()], maxfeerate=0, ) @@ -192,7 +192,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Skip re-signing the transaction for context independent checks from now on # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(tx.serialize().hex())['hex']))) self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-empty'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-vout-empty'}], rawtxs=[tx.serialize().hex()], ) @@ -200,7 +200,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-oversize'}], rawtxs=[tx.serialize().hex()], ) @@ -208,7 +208,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].nValue *= -1 self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-negative'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-vout-negative'}], rawtxs=[tx.serialize().hex()], ) @@ -217,7 +217,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].nValue = 21000000 * COIN + 1 self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-toolarge'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-vout-toolarge'}], rawtxs=[tx.serialize().hex()], ) @@ -226,7 +226,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout = [tx.vout[0]] * 2 tx.vout[0].nValue = 21000000 * COIN self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-txouttotal-toolarge'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-txouttotal-toolarge'}], rawtxs=[tx.serialize().hex()], ) @@ -234,7 +234,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin = [tx.vin[0]] * 2 self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-inputs-duplicate'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-inputs-duplicate'}], rawtxs=[tx.serialize().hex()], ) @@ -243,7 +243,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): raw_tx_coinbase_spent = node.getrawtransaction(txid=node.decoderawtransaction(hexstring=raw_tx_in_block)['vin'][0]['txid']) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_coinbase_spent))) self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: coinbase'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'coinbase'}], rawtxs=[tx.serialize().hex()], ) @@ -251,19 +251,19 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.nVersion = 3 # A version currently non-standard self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: version'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'version'}], rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_0]) # Some non-standard script self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptpubkey'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'scriptpubkey'}], rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptsig-not-pushonly'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'scriptsig-not-pushonly'}], rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) @@ -271,21 +271,21 @@ class MempoolAcceptanceTest(BitcoinTestFramework): num_scripts = 100000 // len(output_p2sh_burn.serialize()) # Use enough outputs to make the tx too large for our policy tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: tx-size'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'tx-size'}], rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0] = output_p2sh_burn tx.vout[0].nValue -= 1 # Make output smaller, such that it is dust for our policy self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: dust'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'dust'}], rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'\xff']) tx.vout = [tx.vout[0]] * 2 self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: multi-op-return'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'multi-op-return'}], rawtxs=[tx.serialize().hex()], ) @@ -294,7 +294,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin[0].nSequence -= 1 # Should be non-max, so locktime is not ignored tx.nLockTime = node.getblockcount() + 1 self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-final'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'non-final'}], rawtxs=[tx.serialize().hex()], ) @@ -303,7 +303,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin[0].nSequence = 2 # We could include it in the second block mined from now, but not the very next one # Can skip re-signing the tx because of early rejection self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-BIP68-final'}], + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'non-BIP68-final'}], rawtxs=[tx.serialize().hex()], maxfeerate=0, ) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index c7d241503a..7014105d88 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -14,13 +14,19 @@ from test_framework.util import ( satoshi_round, ) +# default limits MAX_ANCESTORS = 25 MAX_DESCENDANTS = 25 +# custom limits for node1 +MAX_ANCESTORS_CUSTOM = 5 class MempoolPackagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.extra_args = [["-maxorphantx=1000"], ["-maxorphantx=1000", "-limitancestorcount=5"]] + self.extra_args = [ + ["-maxorphantx=1000"], + ["-maxorphantx=1000", "-limitancestorcount={}".format(MAX_ANCESTORS_CUSTOM)], + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -188,7 +194,14 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 2000) assert_equal(mempool[x]['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) - # TODO: check that node1's mempool is as expected + # Check that node1's mempool is as expected (-> custom ancestor limit) + mempool0 = self.nodes[0].getrawmempool(False) + mempool1 = self.nodes[1].getrawmempool(False) + assert_equal(len(mempool1), MAX_ANCESTORS_CUSTOM) + assert set(mempool1).issubset(set(mempool0)) + for tx in chain[:MAX_ANCESTORS_CUSTOM]: + assert tx in mempool1 + # TODO: more detailed check of node1's mempool (fees etc.) # TODO: test ancestor size limits diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index 7d386c47f6..6676b84e54 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -57,7 +57,7 @@ class RejectLowDifficultyHeadersTest(BitcoinTestFramework): } in self.nodes[0].getchaintips() self.log.info("Feed all fork headers (fails due to checkpoint)") - with self.nodes[0].assert_debug_log(['bad-fork-prior-to-checkpoint (code 67)']): + with self.nodes[0].assert_debug_log(['bad-fork-prior-to-checkpoint']): self.nodes[0].p2p.send_message(msg_headers(self.headers_fork)) self.nodes[0].p2p.wait_for_disconnect() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 58c72f89d8..20864881c1 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" import asyncio -import os import struct import sys @@ -66,27 +65,21 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_at_size = msg_unrecognized(str_data="b" * valid_data_limit) assert len(msg_at_size.serialize()) == msg_limit - increase_allowed = 0.5 - if [s for s in os.environ.get("BITCOIN_CONFIG", "").split(" ") if "--with-sanitizers" in s and "address" in s]: - increase_allowed = 3.5 - with node.assert_memory_usage_stable(increase_allowed=increase_allowed): - self.log.info( - "Sending a bunch of large, junk messages to test " - "memory exhaustion. May take a bit...") + self.log.info("Sending a bunch of large, junk messages to test memory exhaustion. May take a bit...") - # Run a bunch of times to test for memory exhaustion. - for _ in range(80): - node.p2p.send_message(msg_at_size) + # Run a bunch of times to test for memory exhaustion. + for _ in range(80): + node.p2p.send_message(msg_at_size) - # Check that, even though the node is being hammered by nonsense from one - # connection, it can still service other peers in a timely way. - for _ in range(20): - conn2.sync_with_ping(timeout=2) + # Check that, even though the node is being hammered by nonsense from one + # connection, it can still service other peers in a timely way. + for _ in range(20): + conn2.sync_with_ping(timeout=2) - # Peer 1, despite serving up a bunch of nonsense, should still be connected. - self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=320) - assert node.p2p.is_connected + # Peer 1, despite serving up a bunch of nonsense, should still be connected. + self.log.info("Waiting for node to drop junk messages.") + node.p2p.sync_with_ping(timeout=320) + assert node.p2p.is_connected # # 1. @@ -101,11 +94,10 @@ class InvalidMessagesTest(BitcoinTestFramework): msg_over_size = msg_unrecognized(str_data="b" * (valid_data_limit + 1)) assert len(msg_over_size.serialize()) == (msg_limit + 1) - with node.assert_debug_log(["Oversized message from peer=4, disconnecting"]): - # An unknown message type (or *any* message type) over - # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. - node.p2p.send_message(msg_over_size) - node.p2p.wait_for_disconnect(timeout=4) + # An unknown message type (or *any* message type) over + # MAX_PROTOCOL_MESSAGE_LENGTH should result in a disconnect. + node.p2p.send_message(msg_over_size) + node.p2p.wait_for_disconnect(timeout=4) node.disconnect_p2ps() conn = node.add_p2p_connection(P2PDataStore()) @@ -168,7 +160,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_checksum(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - with self.nodes[0].assert_debug_log(['ProcessMessages(badmsg, 2 bytes): CHECKSUM ERROR expected 78df0a04 was ffffffff']): + with self.nodes[0].assert_debug_log(['CHECKSUM ERROR (badmsg, 2 bytes), expected 78df0a04 was ffffffff']): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py new file mode 100755 index 0000000000..7527bdfb08 --- /dev/null +++ b/test/functional/rpc_dumptxoutset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the generation of UTXO snapshots using `dumptxoutset`. +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_raises_rpc_error + +import hashlib +from pathlib import Path + + +class DumptxoutsetTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + """Test a trivial usage of the dumptxoutset RPC command.""" + node = self.nodes[0] + mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 + node.setmocktime(mocktime) + node.generate(100) + + FILENAME = 'txoutset.dat' + out = node.dumptxoutset(FILENAME) + expected_path = Path(node.datadir) / 'regtest' / FILENAME + + assert expected_path.is_file() + + assert_equal(out['coins_written'], 100) + assert_equal(out['base_height'], 100) + assert_equal(out['path'], str(expected_path)) + # Blockhash should be deterministic based on mocked time. + assert_equal( + out['base_hash'], + '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6') + + with open(str(expected_path), 'rb') as f: + digest = hashlib.sha256(f.read()).hexdigest() + # UTXO snapshot hash should be deterministic based on mocked time. + assert_equal( + digest, 'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664') + + # Specifying a path to an existing file will fail. + assert_raises_rpc_error( + -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index c956af1cbe..693051edc0 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -28,6 +28,9 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True + # This test isn't testing tx relay. Set whitelist on the peers for + # instant tx relay. + self.extra_args = [['-whitelist=127.0.0.1']] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -41,6 +44,7 @@ class RawTransactionsTest(BitcoinTestFramework): connect_nodes(self.nodes[0], 3) def run_test(self): + self.log.info("Connect nodes, set fees, generate blocks, and sync") self.min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate @@ -90,7 +94,8 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_option_subtract_fee_from_outputs() def test_change_position(self): - # ensure that setting changePosition in fundraw with an exact match is handled properly + """Ensure setting changePosition in fundraw with an exact match is handled properly.""" + self.log.info("Test fundrawtxn changePosition option") rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) assert_equal(rawmatch["changepos"], -1) @@ -115,9 +120,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() def test_simple(self): - ############### - # simple test # - ############### + self.log.info("Test fundrawtxn") inputs = [ ] outputs = { self.nodes[0].getnewaddress() : 1.0 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) @@ -127,9 +130,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert len(dec_tx['vin']) > 0 #test that we have enough inputs def test_simple_two_coins(self): - ############################## - # simple test with two coins # - ############################## + self.log.info("Test fundrawtxn with 2 coins") inputs = [ ] outputs = { self.nodes[0].getnewaddress() : 2.2 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) @@ -141,9 +142,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') def test_simple_two_outputs(self): - ################################ - # simple test with two outputs # - ################################ + self.log.info("Test fundrawtxn with 2 outputs") + inputs = [ ] outputs = { self.nodes[0].getnewaddress() : 2.6, self.nodes[1].getnewaddress() : 2.5 } rawtx = self.nodes[2].createrawtransaction(inputs, outputs) @@ -159,9 +159,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') def test_change(self): - ######################################################################### - # test a fundrawtransaction with a VIN greater than the required amount # - ######################################################################### + self.log.info("Test fundrawtxn with a vin > required amount") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] @@ -181,9 +179,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(fee + totalOut, utx['amount']) #compare vin total and totalout+fee def test_no_change(self): - ##################################################################### - # test a fundrawtransaction with which will not get a change output # - ##################################################################### + self.log.info("Test fundrawtxn not having a change output") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] @@ -203,9 +199,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(fee + totalOut, utx['amount']) #compare vin total and totalout+fee def test_invalid_option(self): - #################################################### - # test a fundrawtransaction with an invalid option # - #################################################### + self.log.info("Test fundrawtxn with an invalid option") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -220,9 +214,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Unexpected key reserveChangeKey", lambda: self.nodes[2].fundrawtransaction(hexstring=rawtx, options={'reserveChangeKey': True})) def test_invalid_change_address(self): - ############################################################ - # test a fundrawtransaction with an invalid change address # - ############################################################ + self.log.info("Test fundrawtxn with an invalid change address") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -234,9 +226,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "changeAddress must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) def test_valid_change_address(self): - ############################################################ - # test a fundrawtransaction with a provided change address # - ############################################################ + self.log.info("Test fundrawtxn with a provided change address") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -253,9 +243,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(change, out['scriptPubKey']['addresses'][0]) def test_change_type(self): - ######################################################### - # test a fundrawtransaction with a provided change type # - ######################################################### + self.log.info("Test fundrawtxn with a provided change type") utx = get_unspent(self.nodes[2].listunspent(), 5) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']} ] @@ -268,9 +256,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal('witness_v0_keyhash', dec_tx['vout'][rawtx['changepos']]['scriptPubKey']['type']) def test_coin_selection(self): - ######################################################################### - # test a fundrawtransaction with a VIN smaller than the required amount # - ######################################################################### + self.log.info("Test fundrawtxn with a vin < required amount") utx = get_unspent(self.nodes[2].listunspent(), 1) inputs = [ {'txid' : utx['txid'], 'vout' : utx['vout']}] @@ -302,9 +288,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(dec_tx['vout']), 2) def test_two_vin(self): - ########################################### - # test a fundrawtransaction with two VINs # - ########################################### + self.log.info("Test fundrawtxn with 2 vins") utx = get_unspent(self.nodes[2].listunspent(), 1) utx2 = get_unspent(self.nodes[2].listunspent(), 5) @@ -335,9 +319,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(matchingIns, 2) #we now must see two vins identical to vins given as params def test_two_vin_two_vout(self): - ######################################################### - # test a fundrawtransaction with two VINs and two vOUTs # - ######################################################### + self.log.info("Test fundrawtxn with 2 vins and 2 vouts") utx = get_unspent(self.nodes[2].listunspent(), 1) utx2 = get_unspent(self.nodes[2].listunspent(), 5) @@ -360,52 +342,54 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(dec_tx['vout']), 3) def test_invalid_input(self): - ############################################## - # test a fundrawtransaction with invalid vin # - ############################################## + self.log.info("Test fundrawtxn with an invalid vin") inputs = [ {'txid' : "1c7f966dab21119bac53213a2bc7532bff1fa844c124fd750a7d0b1332440bd1", 'vout' : 0} ] #invalid vin! outputs = { self.nodes[0].getnewaddress() : 1.0} rawtx = self.nodes[2].createrawtransaction(inputs, outputs) assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].fundrawtransaction, rawtx) def test_fee_p2pkh(self): - ############################################################ - #compare fee of a standard pubkeyhash transaction + """Compare fee of a standard pubkeyhash transaction.""" + self.log.info("Test fundrawtxn p2pkh fee") inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1.1) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_fee_p2pkh_multi_out(self): - ############################################################ - #compare fee of a standard pubkeyhash transaction with multiple outputs + """Compare fee of a standard pubkeyhash transaction with multiple outputs.""" + self.log.info("Test fundrawtxn p2pkh fee with multiple outputs") inputs = [] - outputs = {self.nodes[1].getnewaddress():1.1,self.nodes[1].getnewaddress():1.2,self.nodes[1].getnewaddress():0.1,self.nodes[1].getnewaddress():1.3,self.nodes[1].getnewaddress():0.2,self.nodes[1].getnewaddress():0.3} + outputs = { + self.nodes[1].getnewaddress():1.1, + self.nodes[1].getnewaddress():1.2, + self.nodes[1].getnewaddress():0.1, + self.nodes[1].getnewaddress():1.3, + self.nodes[1].getnewaddress():0.2, + self.nodes[1].getnewaddress():0.3, + } rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendmany("", outputs) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_fee_p2sh(self): - ############################################################ - #compare fee of a 2of2 multisig p2sh transaction - - # create 2of2 addr + """Compare fee of a 2-of-2 multisig p2sh transaction.""" + # Create 2-of-2 addr. addr1 = self.nodes[1].getnewaddress() addr2 = self.nodes[1].getnewaddress() @@ -419,20 +403,19 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_fee_4of5(self): - ############################################################ - #compare fee of a standard pubkeyhash transaction + """Compare fee of a standard pubkeyhash transaction.""" + self.log.info("Test fundrawtxn fee with 4-of-5 addresses") - # create 4of5 addr + # Create 4-of-5 addr. addr1 = self.nodes[1].getnewaddress() addr2 = self.nodes[1].getnewaddress() addr3 = self.nodes[1].getnewaddress() @@ -445,40 +428,52 @@ class RawTransactionsTest(BitcoinTestFramework): addr4Obj = self.nodes[1].getaddressinfo(addr4) addr5Obj = self.nodes[1].getaddressinfo(addr5) - mSigObj = self.nodes[1].addmultisigaddress(4, [addr1Obj['pubkey'], addr2Obj['pubkey'], addr3Obj['pubkey'], addr4Obj['pubkey'], addr5Obj['pubkey']])['address'] + mSigObj = self.nodes[1].addmultisigaddress( + 4, + [ + addr1Obj['pubkey'], + addr2Obj['pubkey'], + addr3Obj['pubkey'], + addr4Obj['pubkey'], + addr5Obj['pubkey'], + ] + )['address'] inputs = [] outputs = {mSigObj:1.1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) fundedTx = self.nodes[0].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance - ############################################################ def test_spend_2of2(self): - ############################################################ - # spend a 2of2 multisig transaction over fundraw + """Spend a 2-of-2 multisig transaction over fundraw.""" + self.log.info("Test fundrawtxn spending 2-of-2 multisig") - # create 2of2 addr + # Create 2-of-2 addr. addr1 = self.nodes[2].getnewaddress() addr2 = self.nodes[2].getnewaddress() addr1Obj = self.nodes[2].getaddressinfo(addr1) addr2Obj = self.nodes[2].getaddressinfo(addr2) - mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']])['address'] - + mSigObj = self.nodes[2].addmultisigaddress( + 2, + [ + addr1Obj['pubkey'], + addr2Obj['pubkey'], + ] + )['address'] - # send 1.2 BTC to msig addr + # Send 1.2 BTC to msig addr. self.nodes[0].sendtoaddress(mSigObj, 1.2) - self.sync_all() - self.nodes[1].generate(1) + self.nodes[0].generate(1) self.sync_all() oldBalance = self.nodes[1].getbalance() @@ -489,35 +484,18 @@ class RawTransactionsTest(BitcoinTestFramework): signedTx = self.nodes[2].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[2].sendrawtransaction(signedTx['hex']) - self.sync_all() - self.nodes[1].generate(1) + self.nodes[2].generate(1) self.sync_all() - # make sure funds are received at node1 + # Make sure funds are received at node1. assert_equal(oldBalance+Decimal('1.10000000'), self.nodes[1].getbalance()) def test_locked_wallet(self): - ############################################################ - # locked wallet test - self.nodes[1].encryptwallet("test") - self.stop_nodes() - - self.start_nodes() - # This test is not meant to test fee estimation and we'd like - # to be sure all txs are sent at a consistent desired feerate - for node in self.nodes: - node.settxfee(self.min_relay_tx_fee) + self.log.info("Test fundrawtxn with locked wallet") - connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[1], 2) - connect_nodes(self.nodes[0], 2) - connect_nodes(self.nodes[0], 3) - # Again lock the watchonly UTXO or nodes[0] may spend it, because - # lockunspent is memory-only and thus lost on restart - self.nodes[0].lockunspent(False, [{"txid": self.watchonly_txid, "vout": self.watchonly_vout}]) - self.sync_all() + self.nodes[1].encryptwallet("test") - # drain the keypool + # Drain the keypool. self.nodes[1].getnewaddress() self.nodes[1].getrawchangeaddress() inputs = [] @@ -527,7 +505,7 @@ class RawTransactionsTest(BitcoinTestFramework): # creating the key must be impossible because the wallet is locked assert_raises_rpc_error(-4, "Keypool ran out, please call keypoolrefill first", self.nodes[1].fundrawtransaction, rawtx) - #refill the keypool + # Refill the keypool. self.nodes[1].walletpassphrase("test", 100) self.nodes[1].keypoolrefill(8) #need to refill the keypool to get an internal change address self.nodes[1].walletlock() @@ -541,25 +519,23 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[1].createrawtransaction(inputs, outputs) fundedTx = self.nodes[1].fundrawtransaction(rawtx) - #now we need to unlock + # Now we need to unlock. self.nodes[1].walletpassphrase("test", 600) signedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(signedTx['hex']) self.nodes[1].generate(1) self.sync_all() - # make sure funds are received at node1 + # Make sure funds are received at node1. assert_equal(oldBalance+Decimal('51.10000000'), self.nodes[0].getbalance()) def test_many_inputs_fee(self): - ############################################### - # multiple (~19) inputs tx test | Compare fee # - ############################################### + """Multiple (~19) inputs tx test | Compare fee.""" + self.log.info("Test fundrawtxn fee with many inputs") - #empty node1, send some small coins from node0 to node1 + # Empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) - self.sync_all() - self.nodes[0].generate(1) + self.nodes[1].generate(1) self.sync_all() for i in range(0,20): @@ -567,29 +543,27 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() - #fund a tx with ~20 small inputs + # Fund a tx with ~20 small inputs. inputs = [] outputs = {self.nodes[0].getnewaddress():0.15,self.nodes[0].getnewaddress():0.04} rawtx = self.nodes[1].createrawtransaction(inputs, outputs) fundedTx = self.nodes[1].fundrawtransaction(rawtx) - #create same transaction over sendtoaddress + # Create same transaction over sendtoaddress. txId = self.nodes[1].sendmany("", outputs) signedFee = self.nodes[1].getrawmempool(True)[txId]['fee'] - #compare fee + # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance * 19 #~19 inputs def test_many_inputs_send(self): - ############################################# - # multiple (~19) inputs tx test | sign/send # - ############################################# + """Multiple (~19) inputs tx test | sign/send.""" + self.log.info("Test fundrawtxn sign+send with many inputs") - #again, empty node1, send some small coins from node0 to node1 + # Again, empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) - self.sync_all() - self.nodes[0].generate(1) + self.nodes[1].generate(1) self.sync_all() for i in range(0,20): @@ -597,7 +571,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() - #fund a tx with ~20 small inputs + # Fund a tx with ~20 small inputs. oldBalance = self.nodes[0].getbalance() inputs = [] @@ -606,15 +580,12 @@ class RawTransactionsTest(BitcoinTestFramework): fundedTx = self.nodes[1].fundrawtransaction(rawtx) fundedAndSignedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(fundedAndSignedTx['hex']) - self.sync_all() - self.nodes[0].generate(1) + self.nodes[1].generate(1) self.sync_all() assert_equal(oldBalance+Decimal('50.19000000'), self.nodes[0].getbalance()) #0.19+block reward def test_op_return(self): - ##################################################### - # test fundrawtransaction with OP_RETURN and no vin # - ##################################################### + self.log.info("Test fundrawtxn with OP_RETURN and no vin") rawtx = "0100000000010000000000000000066a047465737400000000" dec_tx = self.nodes[2].decoderawtransaction(rawtx) @@ -629,9 +600,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(dec_tx['vout']), 2) # one change output added def test_watchonly(self): - ################################################## - # test a fundrawtransaction using only watchonly # - ################################################## + self.log.info("Test fundrawtxn using only watchonly") inputs = [] outputs = {self.nodes[2].getnewaddress(): self.watchonly_amount / 2} @@ -646,15 +615,13 @@ class RawTransactionsTest(BitcoinTestFramework): assert_greater_than(result["changepos"], -1) def test_all_watched_funds(self): - ############################################################### - # test fundrawtransaction using the entirety of watched funds # - ############################################################### + self.log.info("Test fundrawtxn using entirety of watched funds") inputs = [] outputs = {self.nodes[2].getnewaddress(): self.watchonly_amount} rawtx = self.nodes[3].createrawtransaction(inputs, outputs) - # Backward compatibility test (2nd param is includeWatching) + # Backward compatibility test (2nd param is includeWatching). result = self.nodes[3].fundrawtransaction(rawtx, True) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 2) @@ -673,11 +640,9 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() def test_option_feerate(self): - ####################### - # Test feeRate option # - ####################### + self.log.info("Test fundrawtxn feeRate option") - # Make sure there is exactly one input so coin selection can't skew the result + # Make sure there is exactly one input so coin selection can't skew the result. assert_equal(len(self.nodes[3].listunspent(1)), 1) inputs = [] @@ -692,9 +657,8 @@ class RawTransactionsTest(BitcoinTestFramework): assert_fee_amount(result3['fee'], count_bytes(result3['hex']), 10 * result_fee_rate) def test_address_reuse(self): - ################################ - # Test no address reuse occurs # - ################################ + """Test no address reuse occurs.""" + self.log.info("Test fundrawtxn does not reuse addresses") rawtx = self.nodes[3].createrawtransaction(inputs=[], outputs={self.nodes[3].getnewaddress(): 1}) result3 = self.nodes[3].fundrawtransaction(rawtx) @@ -705,15 +669,13 @@ class RawTransactionsTest(BitcoinTestFramework): changeaddress += out['scriptPubKey']['addresses'][0] assert changeaddress != "" nextaddr = self.nodes[3].getnewaddress() - # Now the change address key should be removed from the keypool + # Now the change address key should be removed from the keypool. assert changeaddress != nextaddr def test_option_subtract_fee_from_outputs(self): - ###################################### - # Test subtractFeeFromOutputs option # - ###################################### + self.log.info("Test fundrawtxn subtractFeeFromOutputs option") - # Make sure there is exactly one input so coin selection can't skew the result + # Make sure there is exactly one input so coin selection can't skew the result. assert_equal(len(self.nodes[3].listunspent(1)), 1) inputs = [] @@ -744,38 +706,39 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[3].createrawtransaction(inputs, outputs) result = [self.nodes[3].fundrawtransaction(rawtx), - # split the fee between outputs 0, 2, and 3, but not output 1 + # Split the fee between outputs 0, 2, and 3, but not output 1. self.nodes[3].fundrawtransaction(rawtx, {"subtractFeeFromOutputs": [0, 2, 3]})] dec_tx = [self.nodes[3].decoderawtransaction(result[0]['hex']), self.nodes[3].decoderawtransaction(result[1]['hex'])] - # Nested list of non-change output amounts for each transaction + # Nested list of non-change output amounts for each transaction. output = [[out['value'] for i, out in enumerate(d['vout']) if i != r['changepos']] for d, r in zip(dec_tx, result)] - # List of differences in output amounts between normal and subtractFee transactions + # List of differences in output amounts between normal and subtractFee transactions. share = [o0 - o1 for o0, o1 in zip(output[0], output[1])] - # output 1 is the same in both transactions + # Output 1 is the same in both transactions. assert_equal(share[1], 0) - # the other 3 outputs are smaller as a result of subtractFeeFromOutputs + # The other 3 outputs are smaller as a result of subtractFeeFromOutputs. assert_greater_than(share[0], 0) assert_greater_than(share[2], 0) assert_greater_than(share[3], 0) - # outputs 2 and 3 take the same share of the fee + # Outputs 2 and 3 take the same share of the fee. assert_equal(share[2], share[3]) - # output 0 takes at least as much share of the fee, and no more than 2 satoshis more, than outputs 2 and 3 + # Output 0 takes at least as much share of the fee, and no more than 2 + # satoshis more, than outputs 2 and 3. assert_greater_than_or_equal(share[0], share[2]) assert_greater_than_or_equal(share[2] + Decimal(2e-8), share[0]) - # the fee is the same in both transactions + # The fee is the same in both transactions. assert_equal(result[0]['fee'], result[1]['fee']) - # the total subtracted from the outputs is equal to the fee + # The total subtracted from the outputs is equal to the fee. assert_equal(share[0] + share[2] + share[3], result[0]['fee']) if __name__ == '__main__': diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index 595b40f7cb..1fdc134f97 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -5,7 +5,7 @@ """Test the invalidateblock RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR from test_framework.util import ( assert_equal, connect_nodes, @@ -62,7 +62,7 @@ class InvalidateTest(BitcoinTestFramework): wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") - blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-1]) @@ -74,7 +74,7 @@ class InvalidateTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) self.log.info("Verify that we reconsider all descendants") - blocks = self.nodes[1].generatetoaddress(10, ADDRESS_BCRT1_UNSPENDABLE) + blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-2]) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 8a3f8c6f06..3da9f05ca5 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -23,6 +23,13 @@ class RpcMiscTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] + self.log.info("test CHECK_NONFATAL") + assert_raises_rpc_error( + -1, + "Internal bug detected: 'request.params.size() != 100'", + lambda: node.echo(*[0] * 100), + ) + self.log.info("test getmemoryinfo") memory = node.getmemoryinfo()['locked'] assert_greater_than(memory['used'], 0) diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index ca0d47a5f8..4ee46d5f53 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -209,7 +209,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtx = self.nodes[2].signrawtransactionwithwallet(rawtx) # This will raise an exception since there are missing inputs - assert_raises_rpc_error(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) + assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx['hex']) ##################################### # getrawtransaction with block hash # @@ -454,7 +454,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Thus, testmempoolaccept should reject testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']], 0.00001000)[0] assert_equal(testres['allowed'], False) - assert_equal(testres['reject-reason'], '256: absurdly-high-fee') + assert_equal(testres['reject-reason'], 'absurdly-high-fee') # and sendrawtransaction should throw assert_raises_rpc_error(-26, "absurdly-high-fee", self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) # and the following calls should both succeed @@ -478,7 +478,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Thus, testmempoolaccept should reject testres = self.nodes[2].testmempoolaccept([rawTxSigned['hex']])[0] assert_equal(testres['allowed'], False) - assert_equal(testres['reject-reason'], '256: absurdly-high-fee') + assert_equal(testres['reject-reason'], 'absurdly-high-fee') # and sendrawtransaction should throw assert_raises_rpc_error(-26, "absurdly-high-fee", self.nodes[2].sendrawtransaction, rawTxSigned['hex']) # and the following calls should both succeed diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md new file mode 100644 index 0000000000..f6ea9ef682 --- /dev/null +++ b/test/functional/test-shell.md @@ -0,0 +1,186 @@ +Test Shell for Interactive Environments +========================================= + +This document describes how to use the `TestShell` submodule in the functional +test suite. + +The `TestShell` submodule extends the `BitcoinTestFramework` functionality to +external interactive environments for prototyping and educational purposes. Just +like `BitcoinTestFramework`, the `TestShell` allows the user to: + +* Manage regtest bitcoind subprocesses. +* Access RPC interfaces of the underlying bitcoind instances. +* Log events to the functional test logging utility. + +The `TestShell` can be useful in interactive environments where it is necessary +to extend the object lifetime of the underlying `BitcoinTestFramework` between +user inputs. Such environments include the Python3 command line interpreter or +[Jupyter](https://jupyter.org/) notebooks running a Python3 kernel. + +## 1. Requirements + +* Python3 +* `bitcoind` built in the same repository as the `TestShell`. + +## 2. Importing `TestShell` from the Bitcoin Core repository + +We can import the `TestShell` by adding the path of the Bitcoin Core +`test_framework` module to the beginning of the PATH variable, and then +importing the `TestShell` class from the `test_shell` sub-package. + +``` +>>> import sys +>>> sys.path.insert(0, "/path/to/bitcoin/test/functional") +>>> from test_framework.test_shell import TestShell +``` + +The following `TestShell` methods manage the lifetime of the underlying bitcoind +processes and logging utilities. + +* `TestShell.setup()` +* `TestShell.shutdown()` + +The `TestShell` inherits all `BitcoinTestFramework` members and methods, such +as: +* `TestShell.nodes[index].rpc_method()` +* `TestShell.log.info("Custom log message")` + +The following sections demonstrate how to initialize, run, and shut down a +`TestShell` object. + +## 3. Initializing a `TestShell` object + +``` +>>> test = TestShell().setup(num_nodes=2, setup_clean_chain=True) +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Initializing test directory /path/to/bitcoin_func_test_XXXXXXX +``` +The `TestShell` forwards all functional test parameters of the parent +`BitcoinTestFramework` object. The full set of argument keywords which can be +used to initialize the `TestShell` can be found in [section +#6](#custom-testshell-parameters) of this document. + +**Note: Running multiple instances of `TestShell` is not allowed.** Running a +single process also ensures that logging remains consolidated in the same +temporary folder. If you need more bitcoind nodes than set by default (1), +simply increase the `num_nodes` parameter during setup. + +``` +>>> test2 = TestShell().setup() +TestShell is already running! +``` + +## 4. Interacting with the `TestShell` + +Unlike the `BitcoinTestFramework` class, the `TestShell` keeps the underlying +Bitcoind subprocesses (nodes) and logging utilities running until the user +explicitly shuts down the `TestShell` object. + +During the time between the `setup` and `shutdown` calls, all `bitcoind` node +processes and `BitcoinTestFramework` convenience methods can be accessed +interactively. + +**Example: Mining a regtest chain** + +By default, the `TestShell` nodes are initialized with a clean chain. This means +that each node of the `TestShell` is initialized with a block height of 0. + +``` +>>> test.nodes[0].getblockchaininfo()["blocks"] +0 +``` + +We now let the first node generate 101 regtest blocks, and direct the coinbase +rewards to a wallet address owned by the mining node. + +``` +>>> address = test.nodes[0].getnewaddress() +>>> test.nodes[0].generatetoaddress(101, address) +['2b98dd0044aae6f1cca7f88a0acf366a4bfe053c7f7b00da3c0d115f03d67efb', ... +``` +Since the two nodes are both initialized by default to establish an outbound +connection to each other during `setup`, the second node's chain will include +the mined blocks as soon as they propagate. + +``` +>>> test.nodes[1].getblockchaininfo()["blocks"] +101 +``` +The block rewards from the first block are now spendable by the wallet of the +first node. + +``` +>>> test.nodes[0].getbalance() +Decimal('50.00000000') +``` + +We can also log custom events to the logger. + +``` +>>> test.nodes[0].log.info("Successfully mined regtest chain!") +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework.node0 (INFO): Successfully mined regtest chain! +``` + +**Note: Please also consider the functional test +[readme](../test/functional/README.md), which provides an overview of the +test-framework**. Modules such as +[key.py](../test/functional/test_framework/key.py), +[script.py](../test/functional/test_framework/script.py) and +[messages.py](../test/functional/test_framework/messages.py) are particularly +useful in constructing objects which can be passed to the bitcoind nodes managed +by a running `TestShell` object. + +## 5. Shutting the `TestShell` down + +Shutting down the `TestShell` will safely tear down all running bitcoind +instances and remove all temporary data and logging directories. + +``` +>>> test.shutdown() +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Cleaning up /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` +To prevent the logs from being removed after a shutdown, simply set the +`TestShell.options.nocleanup` member to `True`. +``` +>>> test.options.nocleanup = True +>>> test.shutdown() +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Stopping nodes +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Not cleaning up dir /path/to/bitcoin_func_test_XXXXXXX on exit +20XX-XX-XXTXX:XX:XX.XXXXXXX TestFramework (INFO): Tests successful +``` + +The following utility consolidates logs from the bitcoind nodes and the +underlying `BitcoinTestFramework`: + +* `/path/to/bitcoin/test/functional/combine_logs.py + '/path/to/bitcoin_func_test_XXXXXXX'` + +## 6. Custom `TestShell` parameters + +The `TestShell` object initializes with the default settings inherited from the +`BitcoinTestFramework` class. The user can override these in +`TestShell.setup(key=value)`. + +**Note:** `TestShell.reset()` will reset test parameters to default values and +can be called after the TestShell is shut down. + +| Test parameter key | Default Value | Description | +|---|---|---| +| `bind_to_localhost_only` | `True` | Binds bitcoind RPC services to `127.0.0.1` if set to `True`.| +| `cachedir` | `"/path/to/bitcoin/test/cache"` | Sets the bitcoind datadir directory. | +| `chain` | `"regtest"` | Sets the chain-type for the underlying test bitcoind processes. | +| `configfile` | `"/path/to/bitcoin/test/config.ini"` | Sets the location of the test framework config file. | +| `coveragedir` | `None` | Records bitcoind RPC test coverage into this directory if set. | +| `loglevel` | `INFO` | Logs events at this level and higher. Can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR` or `CRITICAL`. | +| `nocleanup` | `False` | Cleans up temporary test directory if set to `True` during `shutdown`. | +| `noshutdown` | `False` | Does not stop bitcoind instances after `shutdown` if set to `True`. | +| `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | +| `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | +| `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | +| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | +| `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | +| `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | +| `trace_rpc` | `False` | Logs all RPC calls if set to `True`. | +| `usecli` | `False` | Uses the bitcoin-cli interface for all bitcoind commands instead of directly calling the RPC server. Requires `supports_cli`. | diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 194f2f061b..97585fe054 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -12,6 +12,7 @@ from .util import hex_str_to_bytes from . import segwit_addr ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj' +ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97' class AddressType(enum.Enum): diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index f95c158a68..a9e669fea9 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -478,7 +478,8 @@ class NetworkThread(threading.Thread): wait_until(lambda: not self.network_event_loop.is_running(), timeout=timeout) self.network_event_loop.close() self.join(timeout) - + # Safe to remove event loop. + NetworkThread.network_event_loop = None class P2PDataStore(P2PInterface): """A P2P data store class. diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 780aa5fe03..c56c0d06ff 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -99,12 +99,39 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = False self.bind_to_localhost_only = True self.set_test_params() - - assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + self.parse_args() def main(self): """Main function. This should not be overridden by the subclass test scripts.""" + assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" + + try: + self.setup() + self.run_test() + except JSONRPCException: + self.log.exception("JSONRPC error") + self.success = TestStatus.FAILED + except SkipTest as e: + self.log.warning("Test Skipped: %s" % e.message) + self.success = TestStatus.SKIPPED + except AssertionError: + self.log.exception("Assertion failed") + self.success = TestStatus.FAILED + except KeyError: + self.log.exception("Key error") + self.success = TestStatus.FAILED + except Exception: + self.log.exception("Unexpected exception caught during testing") + self.success = TestStatus.FAILED + except KeyboardInterrupt: + self.log.warning("Exiting after keyboard interrupt") + self.success = TestStatus.FAILED + finally: + exit_code = self.shutdown() + sys.exit(exit_code) + + def parse_args(self): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") @@ -135,6 +162,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.add_options(parser) self.options = parser.parse_args() + def setup(self): + """Call this method to start up the test framework object with options set.""" + PortSeed.n = self.options.port_seed check_json_precision() @@ -181,33 +211,20 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.network_thread = NetworkThread() self.network_thread.start() - success = TestStatus.FAILED + if self.options.usecli: + if not self.supports_cli: + raise SkipTest("--usecli specified but test does not support using CLI") + self.skip_if_no_cli() + self.skip_test_if_missing_module() + self.setup_chain() + self.setup_network() - try: - if self.options.usecli: - if not self.supports_cli: - raise SkipTest("--usecli specified but test does not support using CLI") - self.skip_if_no_cli() - self.skip_test_if_missing_module() - self.setup_chain() - self.setup_network() - self.run_test() - success = TestStatus.PASSED - except JSONRPCException: - self.log.exception("JSONRPC error") - except SkipTest as e: - self.log.warning("Test Skipped: %s" % e.message) - success = TestStatus.SKIPPED - except AssertionError: - self.log.exception("Assertion failed") - except KeyError: - self.log.exception("Key error") - except Exception: - self.log.exception("Unexpected exception caught during testing") - except KeyboardInterrupt: - self.log.warning("Exiting after keyboard interrupt") + self.success = TestStatus.PASSED - if success == TestStatus.FAILED and self.options.pdbonfailure: + def shutdown(self): + """Call this method to shut down the test framework object.""" + + if self.success == TestStatus.FAILED and self.options.pdbonfailure: print("Testcase failed. Attaching python debugger. Enter ? for help") pdb.set_trace() @@ -225,7 +242,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): should_clean_up = ( not self.options.nocleanup and not self.options.noshutdown and - success != TestStatus.FAILED and + self.success != TestStatus.FAILED and not self.options.perf ) if should_clean_up: @@ -238,20 +255,33 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.log.warning("Not cleaning up dir {}".format(self.options.tmpdir)) cleanup_tree_on_exit = False - if success == TestStatus.PASSED: + if self.success == TestStatus.PASSED: self.log.info("Tests successful") exit_code = TEST_EXIT_PASSED - elif success == TestStatus.SKIPPED: + elif self.success == TestStatus.SKIPPED: self.log.info("Test skipped") exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) exit_code = TEST_EXIT_FAILED - logging.shutdown() + # Logging.shutdown will not remove stream- and filehandlers, so we must + # do it explicitly. Handlers are removed so the next test run can apply + # different log handler settings. + # See: https://docs.python.org/3/library/logging.html#logging.shutdown + for h in list(self.log.handlers): + h.flush() + h.close() + self.log.removeHandler(h) + rpc_logger = logging.getLogger("BitcoinRPC") + for h in list(rpc_logger.handlers): + h.flush() + rpc_logger.removeHandler(h) if cleanup_tree_on_exit: shutil.rmtree(self.options.tmpdir) - sys.exit(exit_code) + + self.nodes.clear() + return exit_code # Methods to override in subclass test scripts. def set_test_params(self): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d8bfdfd040..1c9628264f 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -135,25 +135,6 @@ class TestNode(): assert len(self.PRIV_KEYS) == MAX_NODES return self.PRIV_KEYS[self.index] - def get_mem_rss_kilobytes(self): - """Get the memory usage (RSS) per `ps`. - - Returns None if `ps` is unavailable. - """ - assert self.running - - try: - return int(subprocess.check_output( - ["ps", "h", "-o", "rss", "{}".format(self.process.pid)], - stderr=subprocess.DEVNULL).split()[-1]) - - # Avoid failing on platforms where ps isn't installed. - # - # We could later use something like `psutils` to work across platforms. - except (FileNotFoundError, subprocess.SubprocessError): - self.log.exception("Unable to get memory usage") - return None - def _node_msg(self, msg: str) -> str: """Return a modified msg that identifies this node by its index as a debugging aid.""" return "[node %d] %s" % (self.index, msg) @@ -333,33 +314,6 @@ class TestNode(): self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) @contextlib.contextmanager - def assert_memory_usage_stable(self, *, increase_allowed=0.03): - """Context manager that allows the user to assert that a node's memory usage (RSS) - hasn't increased beyond some threshold percentage. - - Args: - increase_allowed (float): the fractional increase in memory allowed until failure; - e.g. `0.12` for up to 12% increase allowed. - """ - before_memory_usage = self.get_mem_rss_kilobytes() - - yield - - after_memory_usage = self.get_mem_rss_kilobytes() - - if not (before_memory_usage and after_memory_usage): - self.log.warning("Unable to detect memory usage (RSS) - skipping memory check.") - return - - perc_increase_memory_usage = (after_memory_usage / before_memory_usage) - 1 - - if perc_increase_memory_usage > increase_allowed: - self._raise_assertion_error( - "Memory usage increased over threshold of {:.3f}% from {} to {} ({:.3f}%)".format( - increase_allowed * 100, before_memory_usage, after_memory_usage, - perc_increase_memory_usage * 100)) - - @contextlib.contextmanager def profile_with_perf(self, profile_name): """ Context manager that allows easy profiling of node activity using `perf`. diff --git a/test/functional/test_framework/test_shell.py b/test/functional/test_framework/test_shell.py new file mode 100644 index 0000000000..26df128f1f --- /dev/null +++ b/test/functional/test_framework/test_shell.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework + +class TestShell: + """Wrapper Class for BitcoinTestFramework. + + The TestShell class extends the BitcoinTestFramework + rpc & daemon process management functionality to external + python environments. + + It is a singleton class, which ensures that users only + start a single TestShell at a time.""" + + class __TestShell(BitcoinTestFramework): + def set_test_params(self): + pass + + def run_test(self): + pass + + def setup(self, **kwargs): + if self.running: + print("TestShell is already running!") + return + + # Num_nodes parameter must be set + # by BitcoinTestFramework child class. + self.num_nodes = 1 + + # User parameters override default values. + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + elif hasattr(self.options, key): + setattr(self.options, key, value) + else: + raise KeyError(key + " not a valid parameter key!") + + super().setup() + self.running = True + return self + + def shutdown(self): + if not self.running: + print("TestShell is not running!") + else: + super().shutdown() + self.running = False + + def reset(self): + if self.running: + print("Shutdown TestShell before resetting!") + else: + self.num_nodes = None + super().__init__() + + instance = None + + def __new__(cls): + # This implementation enforces singleton pattern, and will return the + # previously initialized instance if available + if not TestShell.instance: + TestShell.instance = TestShell.__TestShell() + TestShell.instance.running = False + return TestShell.instance + + def __getattr__(self, name): + return getattr(self.instance, name) + + def __setattr__(self, name, value): + return setattr(self.instance, name, value) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index cde99a2219..4d7967273a 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -316,6 +316,7 @@ def initialize_datadir(dirname, n, chain): f.write("listenonion=0\n") f.write("printtoconsole=0\n") f.write("upnp=0\n") + f.write("shrinkdebugfile=0\n") os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 9c92091f1d..4156e22720 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -181,6 +181,7 @@ BASE_SCRIPTS = [ 'mining_basic.py', 'wallet_bumpfee.py', 'wallet_bumpfee_totalfee_deprecation.py', + 'wallet_implicitsegwit.py', 'rpc_named_arguments.py', 'wallet_listsinceblock.py', 'p2p_leak.py', @@ -190,6 +191,7 @@ BASE_SCRIPTS = [ 'rpc_uptime.py', 'wallet_resendwallettransactions.py', 'wallet_fallbackfee.py', + 'rpc_dumptxoutset.py', 'feature_minchainwork.py', 'rpc_getblockstats.py', 'wallet_create_tx.py', @@ -197,6 +199,7 @@ BASE_SCRIPTS = [ 'feature_uacomment.py', 'wallet_coinbase_category.py', 'feature_filelock.py', + 'feature_loadblock.py', 'p2p_dos_header_tree.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', @@ -361,9 +364,10 @@ def main(): def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, runs_ci, use_term_control): args = args or [] - # Warn if bitcoind is already running (unix only) + # Warn if bitcoind is already running + # pidof might fail or return an empty string if bitcoind is not running try: - if subprocess.check_output(["pidof", "bitcoind"]) is not None: + if subprocess.check_output(["pidof", "bitcoind"]) not in [b'']: print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) except (OSError, subprocess.SubprocessError): pass diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 3c8064ea2d..16925ea753 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -68,6 +68,9 @@ class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 2 + # This test isn't testing txn relay/timing, so set whitelist on the + # peers for instant txn relay. This speeds up the test run time 2-3x. + self.extra_args = [["-whitelist=127.0.0.1"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -87,6 +90,8 @@ class AvoidReuseTest(BitcoinTestFramework): def test_persistence(self): '''Test that wallet files persist the avoid_reuse flag.''' + self.log.info("Test wallet files persist avoid_reuse flag") + # Configure node 1 to use avoid_reuse self.nodes[1].setwalletflag('avoid_reuse') @@ -109,6 +114,8 @@ class AvoidReuseTest(BitcoinTestFramework): def test_immutable(self): '''Test immutable wallet flags''' + self.log.info("Test immutable wallet flags") + # Attempt to set the disable_private_keys flag; this should not work assert_raises_rpc_error(-8, "Wallet flag is immutable", self.nodes[1].setwalletflag, 'disable_private_keys') @@ -130,6 +137,7 @@ class AvoidReuseTest(BitcoinTestFramework): the avoid_reuse flag set to false. This means the 10 BTC send should succeed, where it fails in test_fund_send_fund_send. ''' + self.log.info("Test fund send fund send dirty") fundaddr = self.nodes[1].getnewaddress() retaddr = self.nodes[0].getnewaddress() @@ -183,6 +191,7 @@ class AvoidReuseTest(BitcoinTestFramework): [1] tries to spend 10 BTC (fails; dirty). [1] tries to spend 4 BTC (succeeds; change address sufficient) ''' + self.log.info("Test fund send fund send") fundaddr = self.nodes[1].getnewaddress() retaddr = self.nodes[0].getnewaddress() diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index c50dcd987a..a5f9a047ed 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -109,13 +109,51 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") + # Before `test_balance()`, we have had two nodes with a balance of 50 + # each and then we: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 60 from node B to node A with fee 0.01 + # + # Then we check the balances: + # + # 1) As is + # 2) With transaction 2 from above with 2x the fee + # + # Prior to #16766, in this situation, the node would immediately report + # a balance of 30 on node B as unconfirmed and trusted. + # + # After #16766, we show that balance as unconfirmed. + # + # The balance is indeed "trusted" and "confirmed" insofar as removing + # the mempool transactions would return at least that much money. But + # the algorithm after #16766 marks it as unconfirmed because the 'taint' + # tracking of transaction trust for summing balances doesn't consider + # which inputs belong to a user. In this case, the change output in + # question could be "destroyed" by replace the 1st transaction above. + # + # The post #16766 behavior is correct; we shouldn't be treating those + # funds as confirmed. If you want to rely on that specific UTXO existing + # which has given you that balance, you cannot, as a third party + # spending the other input would destroy that unconfirmed. + # + # For example, if the test transactions were: + # + # 1) Sent 40 from node A to node B with fee 0.01 + # 2) Sent 10 from node B to node A with fee 0.01 + # + # Then our node would report a confirmed balance of 40 + 50 - 10 = 80 + # BTC, which is more than would be available if transaction 1 were + # replaced. + + def test_balances(*, fee_node_1=0): # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1) # change from node 1's send + assert_equal(self.nodes[1].getbalance(), Decimal('0')) # node 1's send had an unsafe input # Same with minconf=0 assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) - assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('0')) # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago # TODO: fix getbalance tracking of coin spentness depth assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) @@ -125,9 +163,9 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[0].getbalances()['mine']['untrusted_pending'], Decimal('60')) assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('0')) - assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('30') - fee_node_1) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getbalances()['mine']['untrusted_pending'], Decimal('30') - fee_node_1) + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('30') - fee_node_1) test_balances(fee_node_1=Decimal('0.01')) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 0948d47653..9d6aa36c35 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -38,7 +38,7 @@ class BumpFeeTest(BitcoinTestFramework): "-walletrbf={}".format(i), "-mintxfee=0.00002", "-deprecatedrpc=totalFee", - "-addresstype=p2sh-segwit", # TODO update constants in test and remove + "-addresstype=bech32", ] for i in range(self.num_nodes)] def skip_test_if_missing_module(self): @@ -211,15 +211,15 @@ def test_small_output_with_feerate_succeeds(rbf_node, dest_address): # Make sure additional inputs exist rbf_node.generatetoaddress(101, rbf_node.getnewaddress()) rbfid = spend_one_input(rbf_node, dest_address) - original_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] - assert_equal(len(original_input_list), 1) - original_txin = original_input_list[0] + input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + assert_equal(len(input_list), 1) + original_txin = input_list[0] # Keep bumping until we out-spend change output tx_fee = 0 while tx_fee < Decimal("0.0005"): - new_input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] - new_item = list(new_input_list)[0] - assert_equal(len(original_input_list), 1) + input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] + new_item = list(input_list)[0] + assert_equal(len(input_list), 1) assert_equal(original_txin["txid"], new_item["txid"]) assert_equal(original_txin["vout"], new_item["vout"]) rbfid_new_details = rbf_node.bumpfee(rbfid) @@ -246,10 +246,8 @@ def test_dust_to_fee(rbf_node, dest_address): # the bumped tx sets fee=49,900, but it converts to 50,000 rbfid = spend_one_input(rbf_node, dest_address) fulltx = rbf_node.getrawtransaction(rbfid, 1) - # (32-byte p2sh-pwpkh output size + 148 p2pkh spend estimate) * 10k(discard_rate) / 1000 = 1800 - # P2SH outputs are slightly "over-discarding" due to the IsDust calculation assuming it will - # be spent as a P2PKH. - bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 1800}) + # (31-vbyte p2wpkh output size + 67-vbyte p2wpkh spend estimate) * 10k(discard_rate) / 1000 = 980 + bumped_tx = rbf_node.bumpfee(rbfid, {"totalFee": 50000 - 980}) full_bumped_tx = rbf_node.getrawtransaction(bumped_tx["txid"], 1) assert_equal(bumped_tx["fee"], Decimal("0.00050000")) assert_equal(len(fulltx["vout"]), 2) @@ -272,7 +270,9 @@ def test_settxfee(rbf_node, dest_address): def test_maxtxfee_fails(test, rbf_node, dest_address): - test.restart_node(1, ['-maxtxfee=0.00003'] + test.extra_args[1]) + # size of bumped transaction (p2wpkh, 1 input, 2 outputs): 141 vbytes + # expected bumping feerate of 20 sats/vbyte => 141*20 sats = 0.00002820 btc + test.restart_node(1, ['-maxtxfee=0.000025'] + test.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) rbfid = spend_one_input(rbf_node, dest_address) assert_raises_rpc_error(-4, "Unable to create transaction: Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) diff --git a/test/functional/wallet_implicitsegwit.py b/test/functional/wallet_implicitsegwit.py new file mode 100755 index 0000000000..379fa6a12f --- /dev/null +++ b/test/functional/wallet_implicitsegwit.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the wallet implicit segwit feature.""" + +import test_framework.address as address +from test_framework.test_framework import BitcoinTestFramework + +# TODO: Might be nice to test p2pk here too +address_types = ('legacy', 'bech32', 'p2sh-segwit') + +def key_to_address(key, address_type): + if address_type == 'legacy': + return address.key_to_p2pkh(key) + elif address_type == 'p2sh-segwit': + return address.key_to_p2sh_p2wpkh(key) + elif address_type == 'bech32': + return address.key_to_p2wpkh(key) + +def send_a_to_b(receive_node, send_node): + keys = {} + for a in address_types: + a_address = receive_node.getnewaddress(address_type=a) + pubkey = receive_node.getaddressinfo(a_address)['pubkey'] + keys[a] = pubkey + for b in address_types: + b_address = key_to_address(pubkey, b) + send_node.sendtoaddress(address=b_address, amount=1) + return keys + +def check_implicit_transactions(implicit_keys, implicit_node): + # The implicit segwit node allows conversion all possible ways + txs = implicit_node.listtransactions(None, 99999) + for a in address_types: + pubkey = implicit_keys[a] + for b in address_types: + b_address = key_to_address(pubkey, b) + assert(('receive', b_address) in tuple((tx['category'], tx['address']) for tx in txs)) + +class ImplicitSegwitTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.log.info("Manipulating addresses and sending transactions to all variations") + implicit_keys = send_a_to_b(self.nodes[0], self.nodes[1]) + + self.sync_all() + + self.log.info("Checking that transactions show up correctly without a restart") + check_implicit_transactions(implicit_keys, self.nodes[0]) + + self.log.info("Checking that transactions still show up correctly after a restart") + self.restart_node(0) + self.restart_node(1) + + check_implicit_transactions(implicit_keys, self.nodes[0]) + +if __name__ == '__main__': + ImplicitSegwitTest().main() diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 4aeb393255..6f248c9bd3 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -5,6 +5,7 @@ """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import BIP125_SEQUENCE_NUMBER from test_framework.util import ( assert_array_result, assert_equal, @@ -12,6 +13,7 @@ from test_framework.util import ( connect_nodes, ) +from decimal import Decimal class ListSinceBlockTest(BitcoinTestFramework): def set_test_params(self): @@ -33,10 +35,12 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_reorg() self.test_double_spend() self.test_double_send() + self.double_spends_filtered() def test_no_blockhash(self): txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) blockhash, = self.nodes[2].generate(1) + blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() txs = self.nodes[0].listtransactions() @@ -44,6 +48,7 @@ class ListSinceBlockTest(BitcoinTestFramework): "category": "receive", "amount": 1, "blockhash": blockhash, + "blockheight": blockheight, "confirmations": 1, }) assert_equal( @@ -273,7 +278,8 @@ class ListSinceBlockTest(BitcoinTestFramework): self.sync_all() # gettransaction should work for txid1 - self.nodes[0].gettransaction(txid1) + tx1 = self.nodes[0].gettransaction(txid1) + assert_equal(tx1['blockheight'], self.nodes[0].getblockheader(tx1['blockhash'])['height']) # listsinceblock(lastblockhash) should now include txid1 in transactions # as well as in removed @@ -291,5 +297,51 @@ class ListSinceBlockTest(BitcoinTestFramework): if tx['txid'] == txid1: assert_equal(tx['confirmations'], 2) + def double_spends_filtered(self): + ''' + `listsinceblock` was returning conflicted transactions even if they + occurred before the specified cutoff blockhash + ''' + spending_node = self.nodes[2] + dest_address = spending_node.getnewaddress() + + tx_input = dict( + sequence=BIP125_SEQUENCE_NUMBER, **next(u for u in spending_node.listunspent())) + rawtx = spending_node.createrawtransaction( + [tx_input], {dest_address: tx_input["amount"] - Decimal("0.00051000"), + spending_node.getrawchangeaddress(): Decimal("0.00050000")}) + signedtx = spending_node.signrawtransactionwithwallet(rawtx) + orig_tx_id = spending_node.sendrawtransaction(signedtx["hex"]) + original_tx = spending_node.gettransaction(orig_tx_id) + + double_tx = spending_node.bumpfee(orig_tx_id) + + # check that both transactions exist + block_hash = spending_node.listsinceblock( + spending_node.getblockhash(spending_node.getblockcount())) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, True) + assert_equal(double_found, True) + + lastblockhash = spending_node.generate(1)[0] + + # check that neither transaction exists + block_hash = spending_node.listsinceblock(lastblockhash) + original_found = False + double_found = False + for tx in block_hash['transactions']: + if tx['txid'] == original_tx['txid']: + original_found = True + if tx['txid'] == double_tx['txid']: + double_found = True + assert_equal(original_found, False) + assert_equal(double_found, False) + if __name__ == '__main__': ListSinceBlockTest().main() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 997d6e702c..8c44a070b8 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -40,14 +40,15 @@ class ListTransactionsTest(BitcoinTestFramework): {"txid": txid}, {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0}) # mine a block, confirmations should change: - self.nodes[0].generate(1) + blockhash = self.nodes[0].generate(1)[0] + blockheight = self.nodes[0].getblockheader(blockhash)['height'] self.sync_all() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, - {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1}) + {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid}, - {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1}) + {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) # send-to-self: txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index f19cf924d2..fde99fe496 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -78,7 +78,7 @@ def main(): os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), '-help=1', ], - timeout=1, + timeout=10, check=True, stderr=subprocess.PIPE, universal_newlines=True, diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 85e8b841b6..7b5707a17a 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -4,7 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -DIR="$1" +# Strip trailing / from directory path (in case it was added by autocomplete) +DIR="${1%/}" COMMIT="$2" if [ -z "$COMMIT" ]; then COMMIT=HEAD diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index 5bbcae79eb..a4c6f0a8d4 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -20,4 +20,15 @@ if [[ ${OUTPUT} != "" ]]; then EXIT_CODE=1 fi +# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it +# is undesirable to crash the whole program. See: src/util/check.h +# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. +OUTPUT=$(git grep -nE 'assert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") +if [[ ${OUTPUT} != "" ]]; then + echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi + exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 8607fc4371..ccd12b5823 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -30,7 +30,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "policy/fees -> txmempool -> validation -> policy/fees" "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/guiutil" "txmempool -> validation -> validationinterface -> txmempool" - "wallet/ismine -> wallet/wallet -> wallet/ismine" + "wallet/scriptpubkeyman -> wallet/wallet -> wallet/scriptpubkeyman" ) EXIT_CODE=0 diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 9f34d0f4dd..99127e01f8 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -20,6 +20,9 @@ FALSE_POSITIVES = [ ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), ("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), + ("src/wallet/scriptpubkeyman.cpp", "WalletLogPrintf(fmt, parameters...)"), + ("src/wallet/scriptpubkeyman.cpp", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), ] diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh index 632ed7c812..72b6b2e9d5 100755 --- a/test/lint/lint-logs.sh +++ b/test/lint/lint-logs.sh @@ -15,6 +15,7 @@ export LC_ALL=C UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ grep -v '\\n"' | \ + grep -v '\.\.\.' | \ grep -v "/\* Continued \*/" | \ grep -v "LogPrint()" | \ grep -v "LogPrintf()") diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 3c82ec19e3..2b1a1d8fbc 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -82,10 +82,10 @@ enabled=( ) if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed. Install by running \"pip3 install flake8\"" + echo "Skipping Python linting since flake8 is not installed." exit 0 elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8 by running \"pip3 install flake8\"" + echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." exit 0 fi diff --git a/test/lint/lint-spelling.ignore-words.txt b/test/lint/lint-spelling.ignore-words.txt index b08837c1d4..576ae94098 100644 --- a/test/lint/lint-spelling.ignore-words.txt +++ b/test/lint/lint-spelling.ignore-words.txt @@ -1,9 +1,7 @@ -cas hights mor mut objext -unselect useable wit unparseable @@ -13,3 +11,4 @@ errorstring keyserver homogenous setban +hist diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index e70b73e1cc..251a94f7ff 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/"); then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 643272de52..e7c690fabe 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -1,9 +1,19 @@ +# -fsanitize=undefined suppressions +# ================================= alignment:move.h alignment:prevector.h -bool:wallet/wallet.cpp float-divide-by-zero:policy/fees.cpp float-divide-by-zero:validation.cpp float-divide-by-zero:wallet/wallet.cpp + +# -fsanitize=integer suppressions +# =============================== +# Unsigned integer overflow occurs when the result of an unsigned integer +# computation cannot be represented in its type. Unlike signed integer overflow, +# this is not undefined behavior, but it is often unintentional. The list below +# contains files in which we expect unsigned integer overflows to occur. The +# list is used to suppress -fsanitize=integer warnings when running our CI UBSan +# job. unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h @@ -32,4 +42,3 @@ unsigned-integer-overflow:stl_bvector.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp -vptr:fs.cpp |