diff options
538 files changed, 4929 insertions, 2398 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index dacfba658b..777eebd2c3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,6 +13,7 @@ environment: QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' + VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' cache: - C:\tools\vcpkg\installed -> build_msvc\vcpkg-packages.txt - C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in @@ -25,7 +26,7 @@ install: # 1. Check whether the vcpkg install directory exists (note that updating the vcpkg-packages.txt file # will cause the appveyor cache rules to invalidate the directory) # 2. If the directory is missing: -# a. Update the vcpkg source (including port files) and build the vcpkg binary, +# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, # b. Install the missing packages. - ps: | $env:PACKAGES = Get-Content -Path build_msvc\vcpkg-packages.txt @@ -34,6 +35,7 @@ install: cd c:\tools\vcpkg $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. git pull origin master + git checkout $env:VCPKG_COMMIT_ID .\bootstrap-vcpkg.bat Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index dd1003d8e3..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: bitcoin-core-ci - -on: - push: -jobs: - build: - runs-on: windows-latest - env: - PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.4/Qt5.9.8_x64_static_vs2019.zip' - QT_DOWNLOAD_HASH: 'f285cbb02bec3b3f3cc2621e3fa7d5edf0d6a66fa30c57859e583acda954ea80' - QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' - VCPKG_INSTALL_PATH: "$env:VCPKG_INSTALLATION_ROOT/installed" - PLATFORM: x64 - steps: - - uses: actions/checkout@v1 - - - uses: actions/setup-python@v1 - with: - python-version: '3.7' # Needed for PEP 540 - - - uses: actions/cache@v1 - id: vcpkgcache - with: - path: C:/vcpkg/installed - key: ${{ runner.os }}-vcpkg - - - name: Update vcpkg and install packages - if: steps.vcpkgcache.outputs.cache-hit != 'true' - run: | - $env:PACKAGES = Get-Content -Path "$env:GITHUB_WORKSPACE/build_msvc/vcpkg-packages.txt" - Write-Host "vcpkg list: $env:PACKAGES" - cd $env:VCPKG_INSTALLATION_ROOT - git pull origin master - .\bootstrap-vcpkg.bat - .\vcpkg install --triplet $env:PLATFORM-windows-static $env:PACKAGES.split() > $null - - name: Install prebuilt Qt libraries - run: | - if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) { - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - exit 1 - } - } - else { - Write-Host "Qt binaries already present."; - } - - name: Generate project files - run: python build_msvc\msvc-autogen.py - - name: Setup MSBuild.exe - uses: warrenbuckley/Setup-MSBuild@v1 - - name: vcpkg integration - run: C:/vcpkg/vcpkg.exe integrate install - - name: Build - run: msbuild build_msvc\bitcoin.sln /m /v:n /p:Configuration=Release - - name: Run test_bitcoin - shell: cmd - run: src\test_bitcoin.exe -k stdout -e stdout 2> NUL - - name: Run bench_bitcoin - shell: cmd - run: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - - name: bitcoin-util-test - run: python test\util\bitcoin-util-test.py - - name: rpcauth-test - shell: cmd - run: python test\util\rpcauth-test.py - - name: test_runner - shell: cmd - run: | - python test\functional\test_runner.py --ansi --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation diff --git a/.gitignore b/.gitignore index 809e851ff1..db493811c5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,9 @@ src/bitcoind src/bitcoin-cli src/bitcoin-tx src/bitcoin-wallet +src/test/fuzz +!src/test/fuzz/*.* src/test/test_bitcoin -src/test/test_bitcoin_fuzzy src/qt/test/test_bitcoin-qt # autoreconf @@ -28,6 +29,7 @@ build-aux/m4/ltversion.m4 build-aux/missing build-aux/compile build-aux/test-driver +config.cache config.log config.status configure diff --git a/.travis.yml b/.travis.yml index 38ad79af29..fbc81b2614 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,14 +82,14 @@ jobs: - set -o errexit; source ./ci/lint/06_script.sh - stage: test - name: 'ARM [GOAL: install] [bionic] [unit tests, functional tests]' + name: 'ARM [GOAL: install] [buster] [unit tests, functional tests]' arch: arm64 env: >- FILE_ENV="./ci/test/00_setup_env_arm.sh" QEMU_USER_CMD="" # Can run the tests natively without qemu - stage: test - name: 'S390x [GOAL: install] [bionic] [unit tests, functional tests]' + name: 'S390x [GOAL: install] [buster] [unit tests, functional tests]' arch: s390x env: >- FILE_ENV="./ci/test/00_setup_env_s390x.sh" @@ -101,14 +101,9 @@ jobs: FILE_ENV="./ci/test/00_setup_env_win64.sh" - stage: test - name: '32-bit + dash [GOAL: install] [gui]' + name: '32-bit + dash [GOAL: install] [CentOS 7] [gui]' env: >- - FILE_ENV="./ci/test/00_setup_env_i686.sh" - - - stage: test - name: 'x86_64 Linux [GOAL: install] [CentOS 7] [no depends, only system libs]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_centos.sh" + FILE_ENV="./ci/test/00_setup_env_i686_centos.sh" - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [uses qt5 dev package and some depends packages] [unsigned char]' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e34f65728..33c797d799 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,10 +88,10 @@ the pull request affects. Valid areas as: - `refactor` for structural changes that do not change behavior - `rpc`, `rest` or `zmq` for changes to the RPC, REST or ZMQ APIs - `script` for changes to the scripts and tools - - `test` for changes to the bitcoin unit tests or QA tests + - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code - - `build` for changes to the GNU Autotools, reproducible builds or CI code + - `build` for changes to the GNU Autotools or reproducible builds Examples: @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2019 The Bitcoin Core developers -Copyright (c) 2009-2019 Bitcoin Developers +Copyright (c) 2009-2020 The Bitcoin Core developers +Copyright (c) 2009-2020 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/autogen.sh b/autogen.sh index 3e922e7e64..de16260b56 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2013-2016 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/build_msvc/README.md b/build_msvc/README.md index 704470cac8..e5aaf57abf 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -12,7 +12,7 @@ Quick Start The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. ``` -vcpkg install --triplet x64-windows-static boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent zeromq berkeleydb rapidcheck double-conversion +vcpkg install --triplet x64-windows-static berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent[thread] rapidcheck zeromq double-conversion py -3 build_msvc\msvc-autogen.py msbuild /m build_msvc\bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` @@ -41,7 +41,9 @@ Qt --------------------- In order to build the Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. -A prebuilt version of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware this download is NOT an officially sanctioned Bitcoin Core distribution and is provided for developer convenience. It should NOT be used for builds that will be used in a production environment or with real funds. +Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. + +To determine which Qt prebuilt version to download open the `.appveyor.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths. To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. @@ -64,18 +66,20 @@ PS >py -3 msvc-autogen.py - An optional step is to adjust the settings in the build_msvc directory and the common.init.vcxproj file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. -- Build with Visual Studio 2017 or msbuild. +- To build from the command line with the Visual Studio 2017 toolchain use: ``` msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build ``` -- Build with Visual Studio 2019 or msbuild. +- To build from the command line with the Visual Studio 2019 toolchain use: ``` msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` +- Alternatively open the `build_msvc\bitcoin.sln` file in Visual Studio. + AppVeyor --------------------- The .appveyor.yml in the root directory is suitable to perform builds on [AppVeyor](https://www.appveyor.com/) Continuous Integration servers. The simplest way to perform an AppVeyor build is to fork Bitcoin Core and then configure a new AppVeyor Project pointing to the forked repository. diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index dcfc1e1b33..17cd31a52e 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -50,7 +50,7 @@ </ProjectReference> </ItemGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> <ClCompile> <AdditionalIncludeDirectories>$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> @@ -64,7 +64,7 @@ </ResourceCompile> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> <ClCompile> <AdditionalIncludeDirectories>$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 9dff0e316e..5e9715451f 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -46,6 +46,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-qt", "bitcoin-qt\bi EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_util\libtest_util.vcxproj", "{868474FD-35F6-4400-8EED-30A33E7521D4}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -230,6 +232,14 @@ Global {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.Build.0 = Release|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.ActiveCfg = Release|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.Build.0 = Release|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.ActiveCfg = Debug|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.ActiveCfg = Debug|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.Build.0 = Debug|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.ActiveCfg = Release|Win32 + {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index e892765fde..3178f2a3d8 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -1,3 +1,7 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_BITCOIN_CONFIG_H #define BITCOIN_BITCOIN_CONFIG_H @@ -49,9 +53,6 @@ /* define if the Boost::Filesystem library is available */ #define HAVE_BOOST_FILESYSTEM /**/ -/* define if the Boost::PROGRAM_OPTIONS library is available */ -#define HAVE_BOOST_PROGRAM_OPTIONS /**/ - /* define if the Boost::System library is available */ #define HAVE_BOOST_SYSTEM /**/ @@ -179,75 +180,6 @@ /* Define to 1 if you have the <inttypes.h> header file. */ #define HAVE_INTTYPES_H 1 -/* Define to 1 if you have the `advapi32' library (-ladvapi32). */ -#define HAVE_LIBADVAPI32 1 - -/* Define to 1 if you have the `comctl32' library (-lcomctl32). */ -#define HAVE_LIBCOMCTL32 1 - -/* Define to 1 if you have the `comdlg32' library (-lcomdlg32). */ -#define HAVE_LIBCOMDLG32 1 - -/* Define to 1 if you have the `crypt32' library (-lcrypt32). */ -#define HAVE_LIBCRYPT32 1 - -/* Define to 1 if you have the `gdi32' library (-lgdi32). */ -#define HAVE_LIBGDI32 1 - -/* Define to 1 if you have the `imm32' library (-limm32). */ -#define HAVE_LIBIMM32 1 - -/* Define to 1 if you have the `iphlpapi' library (-liphlpapi). */ -#define HAVE_LIBIPHLPAPI 1 - -/* Define to 1 if you have the `kernel32' library (-lkernel32). */ -#define HAVE_LIBKERNEL32 1 - -/* Define to 1 if you have the `mingwthrd' library (-lmingwthrd). */ -#define HAVE_LIBMINGWTHRD 1 - -/* Define to 1 if you have the `mswsock' library (-lmswsock). */ -#define HAVE_LIBMSWSOCK 1 - -/* Define to 1 if you have the `ole32' library (-lole32). */ -#define HAVE_LIBOLE32 1 - -/* Define to 1 if you have the `oleaut32' library (-loleaut32). */ -#define HAVE_LIBOLEAUT32 1 - -/* Define to 1 if you have the `rpcrt4' library (-lrpcrt4). */ -#define HAVE_LIBRPCRT4 1 - -/* Define to 1 if you have the `rt' library (-lrt). */ -/* #undef HAVE_LIBRT */ - -/* Define to 1 if you have the `shell32' library (-lshell32). */ -#define HAVE_LIBSHELL32 1 - -/* Define to 1 if you have the `shlwapi' library (-lshlwapi). */ -#define HAVE_LIBSHLWAPI 1 - -/* Define to 1 if you have the `ssp' library (-lssp). */ -#define HAVE_LIBSSP 1 - -/* Define to 1 if you have the `user32' library (-luser32). */ -#define HAVE_LIBUSER32 1 - -/* Define to 1 if you have the `uuid' library (-luuid). */ -#define HAVE_LIBUUID 1 - -/* Define to 1 if you have the `winmm' library (-lwinmm). */ -#define HAVE_LIBWINMM 1 - -/* Define to 1 if you have the `winspool' library (-lwinspool). */ -#define HAVE_LIBWINSPOOL 1 - -/* Define to 1 if you have the `ws2_32' library (-lws2_32). */ -#define HAVE_LIBWS2_32 1 - -/* Define to 1 if you have the `z ' library (-lz ). */ -#define HAVE_LIBZ_ 1 - /* Define this symbol if you have malloc_info */ /* #undef HAVE_MALLOC_INFO */ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 583cd9f88c..c09997d39d 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -6,6 +6,7 @@ <VCProjectVersion>16.0</VCProjectVersion> <VcpkgTriplet Condition="'$(Platform)'=='Win32'">x86-windows-static</VcpkgTriplet> <VcpkgTriplet Condition="'$(Platform)'=='x64'">x64-windows-static</VcpkgTriplet> + <UseNativeEnvironment>true</UseNativeEnvironment> </PropertyGroup> <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'=='' and !Exists('$(WindowsSdkDir)\DesignTime\CommonConfiguration\Neutral\Windows.props')"> @@ -115,7 +116,7 @@ <Link> <SubSystem>Console</SubSystem> <GenerateDebugInformation>true</GenerateDebugInformation> - <AdditionalDependencies>crypt32.lib;Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> <Lib> <AdditionalOptions>/ignore:4221</AdditionalOptions> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index 3951438408..d99b17d381 100644 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2016-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import os import re diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index bd9b6cc8db..2095c0c321 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -1,11 +1,9 @@ <?xml version="1.0" encoding="utf-8"?> -<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Import Project="..\common.init.vcxproj" /> <Import Project="..\common.qt.init.vcxproj" /> <PropertyGroup Label="Globals"> <ProjectGuid>{51201D5E-D939-4854-AE9D-008F03FF518E}</ProjectGuid> - </PropertyGroup> - <PropertyGroup Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> @@ -66,29 +64,32 @@ </ProjectReference> </ItemGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + + <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> <ClCompile> <AdditionalIncludeDirectories>..\libbitcoin_qt\$(GeneratedFilesOutDir)\..\;$(QtIncludeDir)\QtTest;$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> - <AdditionalDependencies>$(QtReleaseLibaries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies>$(QtLibraryDir)\Qt5Test.lib;$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> <ClCompile> <AdditionalIncludeDirectories>..\libbitcoin_qt\$(GeneratedFilesOutDir)\..\;$(QtIncludeDir)\QtTest;$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalOptions>/ignore:4206</AdditionalOptions> </Link> </ItemDefinitionGroup> - <ItemGroup> <MocTestFiles Include="..\..\src\qt\test\addressbooktests.h" /> <MocTestFiles Include="..\..\src\qt\test\apptests.h" /> <MocTestFiles Include="..\..\src\qt\test\compattests.h" /> - <MocTestFiles Include="..\..\src\qt\test\paymentservertests.h" /> <MocTestFiles Include="..\..\src\qt\test\rpcnestedtests.h" /> <MocTestFiles Include="..\..\src\qt\test\uritests.h" /> <MocTestFiles Include="..\..\src\qt\test\wallettests.h" /> @@ -106,8 +107,6 @@ <RemoveDir Directories="$(GeneratedFilesOutDir)\moc\*" /> <RemoveDir Directories="$(GeneratedFilesOutDir)\moc" /> </Target> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <PropertyGroup> <BuildDependsOn> moccode; @@ -120,4 +119,4 @@ $(CleanDependsOn); </CleanDependsOn> </PropertyGroup> - </Project> +</Project> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 96d20a2720..5c4b777d51 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -61,7 +61,9 @@ </PropertyGroup> <ItemGroup> <JsonTestFile Include="..\..\src\test\data\*.json" /> + <RawTestFile Include="..\..\src\test\data\*.raw" /> </ItemGroup> + <HeaderFromHexdump RawFilePath="%(RawTestFile.FullPath)" HeaderFilePath="%(RawTestFile.FullPath).h" SourceHeader="static unsigned const char %(RawTestFile.Filename)_raw[] = {" SourceFooter="};" /> <HeaderFromHexdump RawFilePath="%(JsonTestFile.FullPath)" HeaderFilePath="%(JsonTestFile.FullPath).h" SourceHeader="namespace json_tests{ static unsigned const char %(JsonTestFile.Filename)[] = {" SourceFooter="};}" /> </Target> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp index 0068f588cc..571d19957f 100644 --- a/build_msvc/testconsensus/testconsensus.cpp +++ b/build_msvc/testconsensus/testconsensus.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <iostream> // bitcoin includes. diff --git a/ci/lint/05_before_script.sh b/ci/lint/05_before_script.sh index 28bcbb47f7..2987812c8e 100755 --- a/ci/lint/05_before_script.sh +++ b/ci/lint/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index c7dea599dc..ae8122f9af 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 1f485fbec4..a008d51523 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -51,7 +51,7 @@ export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 6e2542584c..2a522f5a8f 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -9,11 +9,15 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf # The host arch is unknown, so we run the tests through qemu. # If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-arm -L /usr/arm-linux-gnueabihf/"}" -# We don't know whether the host can run the cross compiled binaries. To run them, either qemu-user or libc6:armhf for -# the target is required, so install both. +if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi export DPKG_ADD_ARCH="armhf" -export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +if [ -n "$QEMU_USER_CMD" ]; then + # Likely cross-compiling, so install the needed gcc and qemu-user + export PACKAGES="$PACKAGES qemu-user" +fi +# Use debian to avoid 404 apt errors when cross compiling +export DOCKER_NAME_TAG="debian:buster" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/00_setup_env_i686.sh b/ci/test/00_setup_env_i686.sh deleted file mode 100644 index 6df65dd4a0..0000000000 --- a/ci/test/00_setup_env_i686.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export HOST=i686-pc-linux-gnu -export PACKAGES="g++-multilib python3-zmq" -export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" -export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh new file mode 100644 index 0000000000..b875dceef0 --- /dev/null +++ b/ci/test/00_setup_env_i686_centos.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export HOST=i686-pc-linux-gnu +export DOCKER_NAME_TAG=centos:7 +export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python36-zmq which patch lbzip2 dash" +export GOAL="install" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" +export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index af166b6ca7..2f3d45f1fc 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" -export OSX_SDK=10.11 +export OSX_SDK=10.14 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/ci/test/00_setup_env_native_centos.sh b/ci/test/00_setup_env_native_centos.sh deleted file mode 100644 index 56b915b6c7..0000000000 --- a/ci/test/00_setup_env_native_centos.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export DOCKER_NAME_TAG=centos:7 -export DOCKER_PACKAGES="gcc-c++ libtool make git python3 python36-zmq" -export PACKAGES="boost-devel libevent-devel libdb4-devel libdb4-cxx-devel miniupnpc-devel zeromq-devel qt5-qtbase-devel qt5-qttools-devel qrencode-devel" -export NO_DEPENDS=1 -export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 637d549553..6452feb5f2 100644 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -9,8 +9,15 @@ export LC_ALL=C.UTF-8 export HOST=s390x-linux-gnu # The host arch is unknown, so we run the tests through qemu. # If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string. -export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-s390x"}" -export PACKAGES="python3-zmq bsdmainutils qemu-user" +if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi +export PACKAGES="python3-zmq" +if [ -n "$QEMU_USER_CMD" ]; then + # Likely cross-compiling, so install the needed gcc and qemu-user + export DPKG_ADD_ARCH="s390x" + export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x" +fi +# Use debian to avoid 404 apt errors +export DOCKER_NAME_TAG="debian:buster" export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=true export GOAL="install" diff --git a/ci/test/03_before_install.sh b/ci/test/03_before_install.sh index 5086114ba1..e939b9eeeb 100755 --- a/ci/test/03_before_install.sh +++ b/ci/test/03_before_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 62f30535cb..4d5859e4d3 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,9 @@ export LC_ALL=C.UTF-8 if [[ $DOCKER_NAME_TAG == centos* ]]; then export LC_ALL=en_US.utf8 fi +if [[ $QEMU_USER_CMD == qemu-s390* ]]; then + export LC_ALL=C +fi if [ "$TRAVIS_OS_NAME" == "osx" ]; then set +o errexit @@ -16,6 +19,7 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then git reset --hard origin/master popd || exit 1 set -o errexit + ${CI_RETRY_EXE} brew unlink python@2 ${CI_RETRY_EXE} brew update # brew upgrade returns an error if any of the packages is already up to date # Failure is safe to ignore, unless we really need an update. @@ -41,7 +45,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BITCOIN_CONFIG|BASE_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env +env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) @@ -72,16 +76,6 @@ else } fi -if [ "$TRAVIS_OS_NAME" == "osx" ]; then - top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" -else - DOCKER_EXEC free -m -h - DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - DOCKER_EXEC echo "Free disk space:" - DOCKER_EXEC df -h -fi - if [ -n "$DPKG_ADD_ARCH" ]; then DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" fi @@ -94,6 +88,16 @@ elif [ "$TRAVIS_OS_NAME" != "osx" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi +if [ "$TRAVIS_OS_NAME" == "osx" ]; then + top -l 1 -s 0 | awk ' /PhysMem/ {print}' + echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" +else + DOCKER_EXEC free -m -h + DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + DOCKER_EXEC echo "Free disk space:" + DOCKER_EXEC df -h +fi + if [ ! -d ${DIR_QA_ASSETS} ]; then DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 51a5830682..d8aa5c87a2 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,5 +25,13 @@ if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) fi if [ -z "$NO_DEPENDS" ]; then - DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + if [[ $DOCKER_NAME_TAG == centos* ]]; then + # CentOS has problems building the depends if the config shell is not explicitely set + # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to + # an error as the first command is executed) + SHELL_OPTS="CONFIG_SHELL=/bin/bash" + else + SHELL_OPTS="CONFIG_SHELL=" + fi + DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 98b75d7497..b6043cd61e 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index e641cef279..537493a710 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,16 +11,7 @@ if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 - for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do - # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do - echo "Wrap $b ..." - DOCKER_EXEC mv "$b" "${b}_orig" - DOCKER_EXEC echo "\#\!/usr/bin/env bash" \> "$b" - DOCKER_EXEC echo "$QEMU_USER_CMD \\\"${b}_orig\\\" \\\"\\\$@\\\"" \>\> "$b" - DOCKER_EXEC chmod +x "$b" - done - done + DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" END_FOLD fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh new file mode 100755 index 0000000000..f1d3088748 --- /dev/null +++ b/ci/test/wrap-qemu.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do + # shellcheck disable=SC2044 + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + echo "Wrap $b ..." + mv "$b" "${b}_orig" + echo '#!/usr/bin/env bash' > "$b" + echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b" + chmod +x "$b" + done +done diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh index d2192061db..6b3e6eb7e7 100755 --- a/ci/test/wrap-valgrind.sh +++ b/ci/test/wrap-valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/configure.ac b/configure.ac index ad76317820..18f3104acb 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2019) +define(_COPYRIGHT_YEAR, 2020) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) AC_INIT([Bitcoin Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_REVISION, m4_if(_CLIENT_VERSION_BUILD, [0], [], _CLIENT_VERSION_BUILD))m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) @@ -329,6 +329,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -342,6 +343,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and @@ -481,29 +483,24 @@ use_pkgconfig=yes case $host in *mingw*) - #pkgconfig does more harm than good with MinGW + dnl pkgconfig does more harm than good with MinGW use_pkgconfig=no TARGET_OS=windows - AC_CHECK_LIB([mingwthrd], [main],, AC_MSG_ERROR(libmingwthrd missing)) - AC_CHECK_LIB([kernel32], [main],, AC_MSG_ERROR(libkernel32 missing)) - AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) - AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) - AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) - AC_CHECK_LIB([winspool], [main],, AC_MSG_ERROR(libwinspool missing)) - AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) - AC_CHECK_LIB([shell32], [main],, AC_MSG_ERROR(libshell32 missing)) - AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) - AC_CHECK_LIB([ole32], [main],, AC_MSG_ERROR(libole32 missing)) - AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) - AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) - AC_CHECK_LIB([rpcrt4], [main],, AC_MSG_ERROR(librpcrt4 missing)) - AC_CHECK_LIB([advapi32], [main],, AC_MSG_ERROR(libadvapi32 missing)) - AC_CHECK_LIB([ws2_32], [main],, AC_MSG_ERROR(libws2_32 missing)) - AC_CHECK_LIB([mswsock], [main],, AC_MSG_ERROR(libmswsock missing)) - AC_CHECK_LIB([shlwapi], [main],, AC_MSG_ERROR(libshlwapi missing)) - AC_CHECK_LIB([iphlpapi], [main],, AC_MSG_ERROR(libiphlpapi missing)) - AC_CHECK_LIB([crypt32], [main],, AC_MSG_ERROR(libcrypt32 missing)) + AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) + AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) + AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) + AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) + AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) + AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW],, AC_MSG_ERROR(libshell32 missing)) + AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) + AC_CHECK_LIB([ole32], [CoCreateInstance],, AC_MSG_ERROR(libole32 missing)) + AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) + AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) + AC_CHECK_LIB([advapi32], [CryptAcquireContextW],, AC_MSG_ERROR(libadvapi32 missing)) + AC_CHECK_LIB([ws2_32], [WSAStartup],, AC_MSG_ERROR(libws2_32 missing)) + AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW],, AC_MSG_ERROR(libshlwapi missing)) + AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses],, AC_MSG_ERROR(libiphlpapi missing)) dnl -static is interpreted by libtool, where it has a different meaning. dnl In libtool-speak, it's -all-static. @@ -693,10 +690,6 @@ AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then - dnl glibc absorbed clock_gettime in 2.17. librt (its previous location) is safe to link - dnl in anyway for back-compat. - AC_CHECK_LIB([rt],[clock_gettime],, AC_MSG_ERROR(librt missing)) - dnl __fdelt_chk's params and return type have changed from long unsigned int to long int. dnl See which one is present here. AC_MSG_CHECKING(__fdelt_chk type) diff --git a/contrib/bitcoin-cli.bash-completion b/contrib/bitcoin-cli.bash-completion index f7f12a2773..ddea58a05c 100644 --- a/contrib/bitcoin-cli.bash-completion +++ b/contrib/bitcoin-cli.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoin-cli(1) -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion index da869fa2c3..ec1d9512d4 100644 --- a/contrib/bitcoind.bash-completion +++ b/contrib/bitcoind.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoind(1) and bitcoin-qt(1) -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 0eccbacb96..581fe712e9 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com> Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2019, Bitcoin Core Developers +Copyright: 2009-2020, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 04fa02484f..515a0d8fc6 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -98,22 +98,26 @@ repository (requires pngcrush). security-check.py and test-security-check.py ============================================ -Perform basic ELF security checks on a series of executables. +Perform basic security checks on a series of executables. symbol-check.py =============== -A script to check that the (Linux) executables produced by gitian only contain -allowed gcc, glibc and libstdc++ version symbols. This makes sure they are -still compatible with the minimum supported Linux distribution versions. +A script to check that the executables produced by gitian only contain +certain symbols and are only linked against allowed libraries. + +For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. +This makes sure they are still compatible with the minimum supported distribution versions. + +For macOS we check that the executables are only linked against libraries we allow. Example usage after a gitian build: find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py -If only supported symbols are used the return value will be 0 and the output will be empty. +If no errors occur the return value will be 0 and the output will be empty. -If there are 'unsupported' symbols, the return value will be 1 a list like this will be printed: +If there are any errors the return value will be 1 and output like this will be printed: .../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14 .../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15 diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index 2e4657f1dd..6afa4351e7 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import sys import re diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 5307b04197..1b71245aab 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -19,6 +19,8 @@ EXCLUDE = [ 'src/qt/bitcoinstrings.cpp', 'src/chainparamsseeds.h', # other external copyrights: + 'src/reverse_iterator.h', + 'src/test/fuzz/FuzzedDataProvider.h', 'src/tinyformat.h', 'test/functional/test_framework/bignum.py', # python init: @@ -455,14 +457,14 @@ CPP_HEADER = ''' def get_cpp_header_lines_to_insert(start_year, end_year): return reversed(get_header_lines(CPP_HEADER, start_year, end_year)) -PYTHON_HEADER = ''' +SCRIPT_HEADER = ''' # Copyright (c) %s The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -def get_python_header_lines_to_insert(start_year, end_year): - return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year)) +def get_script_header_lines_to_insert(start_year, end_year): + return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year)) ################################################################################ # query git for year of last change @@ -491,17 +493,18 @@ def file_has_hashbang(file_lines): return False return file_lines[0][:2] == '#!' -def insert_python_header(filename, file_lines, start_year, end_year): +def insert_script_header(filename, file_lines, start_year, end_year): if file_has_hashbang(file_lines): insert_idx = 1 else: insert_idx = 0 - header_lines = get_python_header_lines_to_insert(start_year, end_year) + header_lines = get_script_header_lines_to_insert(start_year, end_year) for line in header_lines: file_lines.insert(insert_idx, line) write_file_lines(filename, file_lines) def insert_cpp_header(filename, file_lines, start_year, end_year): + file_lines.insert(0, '\n') header_lines = get_cpp_header_lines_to_insert(start_year, end_year) for line in header_lines: file_lines.insert(0, line) @@ -513,8 +516,8 @@ def exec_insert_header(filename, style): sys.exit('*** %s already has a copyright by The Bitcoin Core developers' % (filename)) start_year, end_year = get_git_change_year_range(filename) - if style == 'python': - insert_python_header(filename, file_lines, start_year, end_year) + if style in ['python', 'shell']: + insert_script_header(filename, file_lines, start_year, end_year) else: insert_cpp_header(filename, file_lines, start_year, end_year) @@ -555,11 +558,13 @@ def insert_cmd(argv): if not os.path.isfile(filename): sys.exit("*** bad filename: %s" % filename) _, extension = os.path.splitext(filename) - if extension not in ['.h', '.cpp', '.cc', '.c', '.py']: + if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']: sys.exit("*** cannot insert for file extension %s" % extension) if extension == '.py': style = 'python' + elif extension == '.sh': + style = 'shell' else: style = 'cpp' exec_insert_header(filename, style) diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index dbdb622877..aa65953d83 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2016-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 44b7f6c7cc..21d64e893d 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -Perform basic ELF security checks on a series of executables. +Perform basic security checks on a series of executables. Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. -Needs `readelf` (for ELF) and `objdump` (for PE). +Needs `readelf` (for ELF), `objdump` (for PE) and `otool` (for MACHO). ''' import subprocess import sys @@ -14,6 +14,7 @@ import os READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump') +OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') NONFATAL = {} # checks which are non-fatal for now but only generate a warning def check_ELF_PIE(executable): @@ -162,6 +163,40 @@ def check_PE_NX(executable): (arch,bits) = get_PE_dll_characteristics(executable) return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT +def get_MACHO_executable_flags(executable): + p = subprocess.Popen([OTOOL_CMD, '-vh', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + + flags = [] + for line in stdout.splitlines(): + tokens = line.split() + # filter first two header lines + if 'magic' in tokens or 'Mach' in tokens: + continue + # filter ncmds and sizeofcmds values + flags += [t for t in tokens if not t.isdigit()] + return flags + +def check_MACHO_PIE(executable) -> bool: + ''' + Check for position independent executable (PIE), allowing for address space randomization. + ''' + flags = get_MACHO_executable_flags(executable) + if 'PIE' in flags: + return True + return False + +def check_MACHO_NOUNDEFS(executable) -> bool: + ''' + Check for no undefined references. + ''' + flags = get_MACHO_executable_flags(executable) + if 'NOUNDEFS' in flags: + return True + return False + CHECKS = { 'ELF': [ ('PIE', check_ELF_PIE), @@ -173,6 +208,10 @@ CHECKS = { ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_PE_NX) +], +'MACHO': [ + ('PIE', check_MACHO_PIE), + ('NOUNDEFS', check_MACHO_NOUNDEFS), ] } @@ -183,6 +222,8 @@ def identify_executable(executable): return 'PE' elif magic.startswith(b'\x7fELF'): return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' return None if __name__ == '__main__': diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 0c59ab6239..f92d997621 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,6 +15,7 @@ import subprocess import re import sys import os +from typing import List, Optional, Tuple # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # @@ -52,8 +53,10 @@ IGNORE_EXPORTS = { } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool') + # Allowed NEEDED libraries -ALLOWED_LIBRARIES = { +ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library @@ -79,6 +82,25 @@ ARCH_MIN_GLIBC_VER = { 'AArch64':(2,17), 'RISC-V': (2,27) } + +MACHO_ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libc++.1.dylib', # C++ Standard Library +'libSystem.B.dylib', # libc, libm, libpthread, libinfo +# bitcoin-qt only +'AppKit', # user interface +'ApplicationServices', # common application tasks. +'Carbon', # deprecated c back-compat API +'CoreFoundation', # low level func, data types +'CoreGraphics', # 2D rendering +'CoreServices', # operating system services +'CoreText', # interface for laying out text and handling fonts. +'Foundation', # base layer functionality for apps/frameworks +'ImageIO', # read and write image file formats. +'IOKit', # user-space access to hardware devices and drivers. +'libobjc.A.dylib', # Objective-C runtime library +} + class CPPFilt(object): ''' Demangle C++ symbol names. @@ -98,15 +120,15 @@ class CPPFilt(object): self.proc.stdout.close() self.proc.wait() -def read_symbols(executable, imports=True): +def read_symbols(executable, imports=True) -> List[Tuple[str, str, str]]: ''' - Parse an ELF executable and return a list of (symbol,version) tuples + Parse an ELF executable and return a list of (symbol,version, arch) tuples for dynamic, imported symbols. ''' p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: - raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) + raise IOError('Could not read symbols for {}: {}'.format(executable, stderr.strip())) syms = [] for line in stdout.splitlines(): line = line.split() @@ -121,7 +143,7 @@ def read_symbols(executable, imports=True): syms.append((sym, version, arch)) return syms -def check_version(max_versions, version, arch): +def check_version(max_versions, version, arch) -> bool: if '_' in version: (lib, _, ver) = version.rpartition('_') else: @@ -132,7 +154,7 @@ def check_version(max_versions, version, arch): return False return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch] -def read_libraries(filename): +def elf_read_libraries(filename) -> List[str]: p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) (stdout, stderr) = p.communicate() if p.returncode: @@ -148,26 +170,94 @@ def read_libraries(filename): raise ValueError('Unparseable (NEEDED) specification') return libraries -if __name__ == '__main__': +def check_imported_symbols(filename) -> bool: cppfilt = CPPFilt() + ok = True + for sym, version, arch in read_symbols(filename, True): + if version and not check_version(MAX_VERSIONS, version, arch): + print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) + ok = False + return ok + +def check_exported_symbols(filename) -> bool: + cppfilt = CPPFilt() + ok = True + for sym,version,arch in read_symbols(filename, False): + if arch == 'RISC-V' or sym in IGNORE_EXPORTS: + continue + print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + ok = False + return ok + +def check_ELF_libraries(filename) -> bool: + ok = True + for library_name in elf_read_libraries(filename): + if library_name not in ELF_ALLOWED_LIBRARIES: + print('{}: NEEDED library {} is not allowed'.format(filename, library_name)) + ok = False + return ok + +def macho_read_libraries(filename) -> List[str]: + p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.splitlines(): + tokens = line.split() + if len(tokens) == 1: # skip executable name + continue + libraries.append(tokens[0].split('/')[-1]) + return libraries + +def check_MACHO_libraries(filename) -> bool: + ok = True + for dylib in macho_read_libraries(filename): + if dylib not in MACHO_ALLOWED_LIBRARIES: + print('{} is not in ALLOWED_LIBRARIES!'.format(dylib)) + ok = False + return ok + +CHECKS = { +'ELF': [ + ('IMPORTED_SYMBOLS', check_imported_symbols), + ('EXPORTED_SYMBOLS', check_exported_symbols), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries) +], +'MACHO': [ + ('DYNAMIC_LIBRARIES', check_MACHO_libraries) +] +} + +def identify_executable(executable) -> Optional[str]: + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' + return None + +if __name__ == '__main__': retval = 0 for filename in sys.argv[1:]: - # Check imported symbols - for sym,version,arch in read_symbols(filename, True): - if version and not check_version(MAX_VERSIONS, version, arch): - print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) - retval = 1 - # Check exported symbols - if arch != 'RISC-V': - for sym,version,arch in read_symbols(filename, False): - if sym in IGNORE_EXPORTS: - continue - print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) - retval = 1 - # Check dependency libraries - for library_name in read_libraries(filename): - if library_name not in ALLOWED_LIBRARIES: - print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + try: + etype = identify_executable(filename) + if etype is None: + print('{}: unknown format'.format(filename)) retval = 1 + continue + failed = [] + for (name, func) in CHECKS[etype]: + if not func(filename): + failed.append(name) + if failed: + print('{}: failed {}'.format(filename, ' '.join(failed))) + retval = 1 + except IOError: + print('{}: cannot open'.format(filename)) + retval = 1 sys.exit(retval) diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index bb864bfc0c..438d5f6bf0 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh index 88ac850021..f5cd05a2c3 100755 --- a/contrib/devtools/test_deterministic_coverage.sh +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -21,8 +21,8 @@ NON_DETERMINISTIC_TESTS=( "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() - "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) - "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index df1db76e92..75034616f7 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2017-2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index e38fa6fcb0..4a3df93cea 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse import os @@ -206,7 +209,7 @@ def main(): args.macos = 'm' in args.os # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.14.sdk.tar.gz'): print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') args.macos = False diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 2b86602a82..257dd8ba30 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "linux64" +- "amd64" packages: - "curl" - "g++-aarch64-linux-gnu" diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 2d49493641..a4f3219c22 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -4,7 +4,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "linux64" +- "amd64" packages: - "faketime" remotes: diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index d3a2645c53..2b6aa599e0 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "linux64" +- "amd64" packages: - "ca-certificates" - "curl" @@ -32,7 +32,7 @@ remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: -- "MacOSX10.11.sdk.tar.gz" +- "MacOSX10.14.sdk.tar.gz" script: | set -e -o pipefail @@ -90,7 +90,7 @@ script: | BASEPREFIX="${PWD}/depends" mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz # Build dependencies for each host for i in $HOSTS; do @@ -137,6 +137,8 @@ script: | CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} + make ${MAKEOPTS} -C src check-security + make ${MAKEOPTS} -C src check-symbols make install-strip DESTDIR=${INSTALLPATH} make osx_volname diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 70b7bb111d..9d96465742 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -4,7 +4,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "linux64" +- "amd64" packages: - "libssl-dev" - "autoconf" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index b772404ae5..de2e45190a 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -5,7 +5,7 @@ distro: "ubuntu" suites: - "bionic" architectures: -- "linux64" +- "amd64" packages: - "curl" - "g++" diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 4dfa1729a5..46d755886c 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -62,15 +62,16 @@ Likewise, to perform a bootstrapped build (takes even longer): export ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap --no-substitutes' ``` -### Using the right Guix +### Using a version of Guix with `guix time-machine` capabilities -Once Guix is installed, deploy our patched version into your current Guix -profile. The changes there are slowly being upstreamed. +> Note: This entire section can be skipped if you are already using a version of +> Guix that has [the `guix time-machine` command][guix/time-machine]. + +Once Guix is installed, if it doesn't have the `guix time-machine` command, pull +the latest `guix`. ```sh -guix pull --url=https://github.com/dongcarl/guix.git \ - --commit=82c77e52b8b46e0a3aad2cb12307c2e30547deec \ - --max-jobs=4 # change accordingly +guix pull --max-jobs=4 # change number of jobs accordingly ``` Make sure that you are using your current profile. (You are prompted to do this @@ -80,9 +81,6 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` -> Note: There is ongoing work to eliminate this entire section using Guix -> [inferiors][guix/inferiors] and [channels][guix/channels]. - ## Usage ### As a Development Environment @@ -224,6 +222,7 @@ repository and will likely put one up soon. [guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html [guix/inferiors]: https://www.gnu.org/software/guix/manual/en/html_node/Inferiors.html [guix/channels]: https://www.gnu.org/software/guix/manual/en/html_node/Channels.html +[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html [debian/guix-package]: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=850644 [fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index f8ba8c7ed2..5e0c681f29 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -13,6 +13,12 @@ make -C "${PWD}/depends" -j"$MAX_JOBS" download ${V:+V=1} ${SOURCES_PATH:+SOURCE # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +time-machine() { + guix time-machine --url=https://github.com/dongcarl/guix.git \ + --commit=b3a7c72c8b2425f8ddb0fc6e3b1caeed40f86dee \ + -- "$@" +} + # Deterministically build Bitcoin Core for HOSTs (overriable by environment) for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu}; do @@ -22,18 +28,18 @@ for host in ${HOSTS=i686-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64- # Run the build script 'contrib/guix/libexec/build.sh' in the build # container specified by 'contrib/guix/manifest.scm' # shellcheck disable=SC2086 - guix environment --manifest="${PWD}/contrib/guix/manifest.scm" \ - --container \ - --pure \ - --no-cwd \ - --share="$PWD"=/bitcoin \ - ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ - ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ - -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ - SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ - ${V:+V=1} \ - ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ - bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/bitcoin \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$host" \ + MAX_JOBS="$MAX_JOBS" \ + SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ + ${V:+V=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" done diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 4b848dda99..e9130a21de 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. # Install libdb4.8 (Berkeley DB). diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 2315898bf1..5990b9307a 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -28,6 +28,11 @@ input=/home/example/.bitcoin/blocks #genesis=000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943 #input=/home/example/.bitcoin/testnet3/blocks +# regtest +#netmagic=fabfb5da +#genesis=0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206 +#input=/home/example/.bitcoin/regtest/blocks + # "output" option causes blockchain files to be written to the given location, # with "output_file" ignored. If not used, "output_file" is used instead. # output=/home/example/blockchain_directory diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 863b22f6b1..bcca3b7cea 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2018 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -15,6 +15,7 @@ import sys import hashlib import datetime import time +import glob from collections import namedtuple from binascii import unhexlify @@ -92,6 +93,30 @@ def mkblockmap(blkindex): blkmap[hash] = height return blkmap +# This gets the first block file ID that exists from the input block +# file directory. +def getFirstBlockFileId(block_dir_path): + # First, this sets up a pattern to search for block files, for + # example 'blkNNNNN.dat'. + blkFilePattern = os.path.join(block_dir_path, "blk[0-9][0-9][0-9][0-9][0-9].dat") + + # This search is done with glob + blkFnList = glob.glob(blkFilePattern) + + if len(blkFnList) == 0: + print("blocks not pruned - starting at 0") + return 0 + # We then get the lexicographic minimum, which should be the first + # block file name. + firstBlkFilePath = min(blkFnList) + firstBlkFn = os.path.basename(firstBlkFilePath) + + # now, the string should be ['b','l','k','N','N','N','N','N','.','d','a','t'] + # So get the ID by choosing: 3 4 5 6 7 + # The ID is not necessarily 0 if this is a pruned node. + blkId = int(firstBlkFn[3:8]) + return blkId + # Block header and extent on disk BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size']) @@ -101,7 +126,9 @@ class BlockDataCopier: self.blkindex = blkindex self.blkmap = blkmap - self.inFn = 0 + # Get first occurring block file id - for pruned nodes this + # will not necessarily be 0 + self.inFn = getFirstBlockFileId(self.settings['input']) self.inF = None self.outFn = 0 self.outsz = 0 diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 02c96d2a75..fed6e665b8 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -2,7 +2,7 @@ # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # -# Copyright (c) 2013-2018 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 29b49ebff4..f78bebf114 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -1,15 +1,135 @@ -### MacDeploy ### +# MacOS Deployment -For Snow Leopard (which uses [Python 2.6](http://www.python.org/download/releases/2.6/)), you will need the param_parser package: +The `macdeployqtplus` script should not be run manually. Instead, after building as usual: - sudo easy_install argparse +```bash +make deploy +``` -This script should not be run manually, instead, after building as usual: +During the deployment process, the disk image window will pop up briefly +when the fancy settings are applied. This is normal, please do not interfere, +the process will unmount the DMG and cleanup before finishing. - make deploy +When complete, it will have produced `Bitcoin-Qt.dmg`. -During the process, the disk image window will pop up briefly where the fancy -settings are applied. This is normal, please do not interfere. +## SDK Extraction -When finished, it will produce `Bitcoin-Qt.dmg`. +`Xcode.app` is packaged in a `.xip` archive. +This makes the SDK less-trivial to extract on non-macOS machines. +One approach (tested on Debian Buster) is outlined below: +```bash + +apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make + +git clone https://github.com/tpoechtrager/xar +pushd xar/xar +./configure +make +make install +popd + +git clone https://github.com/NiklasRosenstein/pbzx +pushd pbzx +clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib +popd + +xar -xf Xcode_10.2.1.xip -C . + +./pbzx/pbzx -n Content | cpio -i + +find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \; +``` + +on macOS the process is more straightforward: + +```bash +xip -x Xcode_10.2.1.xip +tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +``` + +Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from +[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). +The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First +ensure the DMG file is in the current directory, and then run the script. You +may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the +directory) when you've confirmed the extraction succeeded. + +```bash +apt-get install p7zip-full sleuthkit +contrib/macdeploy/extract-osx-sdk.sh +rm -rf 5.hfs MacOSX10.11.sdk +``` + +## Deterministic macOS DMG Notes +Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple +`binutils` (`ld`, `ar`, etc) and DMG authoring tools. + +Apple uses `clang` extensively for development and has upstreamed the necessary +functionality so that a vanilla clang can take advantage. It supports the use of `-F`, +`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when +building for macOS. + +Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the +FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several +other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These +do not build under Linux, so they have been patched to do so. The work here was used as +a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). + +In order to build a working toolchain, the following source packages are needed from +Apple: `cctools`, `dyld`, and `ld64`. + +These tools inject timestamps by default, which produce non-deterministic binaries. The +`ZERO_AR_DATE` environment variable is used to disable that. + +This version of `cctools` has been patched to use the current version of `clang`'s headers +and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. + +To complicate things further, all builds must target an Apple SDK. These SDKs are free to +download, but not redistributable. To obtain it, register for an Apple Developer Account, +then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). + +This file is many gigabytes in size, but most (but not all) of what we need is +contained only in a single directory: + +```bash +Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk +``` + +See the SDK Extraction notes above for how to obtain it. + +The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are +created using these tools. The build process has been designed to avoid including the +SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely +redistributed. + +`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been +patched. A system `genisoimage` will work fine, but it will not be deterministic because +the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch). +No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for +a single invocation, so that's no real concern. + +`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its +maintainer has seemingly abandoned the project. + +The DMG tool has the ability to create DMGs from scratch as well, but this functionality is +broken. Only the compression feature is currently used. Ideally, the creation could be fixed +and `genisoimage` would no longer be necessary. + +Background images and other features can be added to DMG files by inserting a +`.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`. + +As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in +order to satisfy the new Gatekeeper requirements. Because this private key cannot be +shared, we'll have to be a bit creative in order for the build process to remain somewhat +deterministic. Here's how it works: + +- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which + users may choose to bless and run. It also outputs an unsigned app structure in the form + of a tarball, which also contains all of the tools that have been previously (deterministically) + built in order to create a final DMG. +- The Apple keyholder uses this unsigned app to create a detached signature, using the + script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Gitian. It uses the + pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index af2b11fa0d..5c5a85d3fe 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 938bcd1638..31a97f0a24 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index 4c175156f4..21243ada04 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index ccb0f4f895..8408545a21 100644 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index ec589d4c02..d516ca10c1 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2013-2018 The Bitcoin Core developers +# Copyright (c) 2013-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -19,13 +19,9 @@ MIN_BLOCKS = 337600 # These are hosts that have been observed to be behaving strangely (e.g. # aggressively connecting to every node). -SUSPICIOUS_HOSTS = { - "130.211.129.106", "178.63.107.226", - "83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6", - "54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211", - "54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214", - "54.94.195.96", "54.94.200.247" -} +with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: + SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} + PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt new file mode 100644 index 0000000000..13385cc816 --- /dev/null +++ b/contrib/seeds/suspicious_hosts.txt @@ -0,0 +1,16 @@ +130.211.129.106 +148.251.238.178 +176.9.46.6 +178.63.107.226 +54.173.72.127 +54.174.10.182 +54.183.64.54 +54.194.231.211 +54.66.214.167 +54.66.220.137 +54.67.33.14 +54.77.251.214 +54.94.195.96 +54.94.200.247 +83.81.130.26 +88.198.17.7
\ No newline at end of file diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index f232bb62c2..744b8ee70f 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -184,3 +184,16 @@ ... fun:_ZN5BCLog6Logger12StartLoggingEv } +{ + Suppress BCLog::Logger::StartLogging() still reachable memory warning + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + ... + fun:_ZN5BCLog6Logger12StartLoggingEv +} +{ + Suppress rest_blockhash_by_height Conditional jump or move depends on uninitialised value(s) + Memcheck:Cond + fun:_ZL24rest_blockhash_by_heightP11HTTPRequestRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE +} diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 288e4ccacb..db5bfce208 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index 4db4a90853..a26791f0d1 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 9ec8663fba..7e46c6fd47 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify commits against a trusted keys list.""" diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index bfe74aa4fa..4296998631 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index cc42422b23..31720e72e7 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1f88c209cf..1bc4fb8189 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,9 +1,8 @@ OSX_MIN_VERSION=10.12 -OSX_SDK_VERSION=10.11 +OSX_SDK_VERSION=10.14 OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk -LD64_VERSION=253.9 -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -stdlib=libc++ +darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) +darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index d360bb5ba6..cd0e70fb1c 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -29,11 +29,11 @@ $(package)_cxxflags_android=-fPIC endef define $(package)_preprocess_cmds - echo "using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$(boost_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam + echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam endef define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$(boost_config_libraries) + ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) endef define $(package)_build_cmds diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index a065256c1c..4195230b40 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -1,45 +1,55 @@ package=native_cctools -$(package)_version=807d6fd1be5d2224872e381870c0a75387fe05e6 -$(package)_download_path=https://github.com/theuni/cctools-port/archive +$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f +$(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a +$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04 $(package)_build_subdir=cctools -$(package)_clang_version=3.7.1 -$(package)_clang_download_path=https://llvm.org/releases/$($(package)_clang_version) +$(package)_clang_version=6.0.1 +$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz -$(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9 +$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0 + +$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef +$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive +$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz +$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz +$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3 + $(package)_extra_sources=$($(package)_clang_file_name) +$(package)_extra_sources += $($(package)_libtapi_file_name) define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ -$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) +$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \ +$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash)) endef define $(package)_extract_cmds mkdir -p $($(package)_extract_dir) && \ echo "$($(package)_sha256_hash) $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \ echo "$($(package)_clang_sha256_hash) $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ + echo "$($(package)_libtapi_sha256_hash) $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ - mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ + mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \ + mkdir -p libtapi && \ + tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \ tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \ rm -f toolchain/lib/libc++abi.so* && \ - echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ - echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ - chmod +x toolchain/bin/$(host)-dsymutil && \ tar --no-same-owner --strip-components=1 -xf $($(package)_source) endef define $(package)_set_vars -$(package)_config_opts=--target=$(host) --disable-lto-support -$(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib -$(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang -$(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ + $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir) + $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib + $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang + $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++ endef define $(package)_preprocess_cmds - cd $($(package)_build_subdir); ./autogen.sh && \ - sed -i.old "/define HAVE_PTHREADS/d" ld64/src/ld/InputFiles.h + CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \ + CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \ + sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h endef define $(package)_config_cmds @@ -52,6 +62,9 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + mkdir -p $($(package)_staging_prefix_dir)/lib/ && \ + cd $($(package)_extract_dir) && \ + cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \ cd $($(package)_extract_dir)/toolchain && \ mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index efa76965d5..366b1d0c42 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -121,7 +121,6 @@ $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSIO $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_darwin += -device-option MAC_MIN_VERSION=$(OSX_MIN_VERSION) $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) -$(package)_config_opts_darwin += -device-option MAC_LD64_VERSION=$(LD64_VERSION) endif $(package)_config_opts_linux = -qt-xkbcommon-x11 diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 337d0eb9ca..4cd96df29f 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -18,7 +18,7 @@ QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS -!host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} -mlinker-version=$${MAC_LD64_VERSION} +!host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} QMAKE_AR = $${CROSS_COMPILE}ar cq QMAKE_RANLIB=$${CROSS_COMPILE}ranlib QMAKE_LIBTOOL=$${CROSS_COMPILE}libtool diff --git a/doc/build-osx.md b/doc/build-osx.md index bf655538c7..86b5c5b602 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -113,96 +113,3 @@ tail -f $HOME/Library/Application\ Support/Bitcoin/debug.log * Tested on OS X 10.12 Sierra through macOS 10.15 Catalina on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714). - -## Deterministic macOS DMG Notes -Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple -`binutils` (`ld`, `ar`, etc) and DMG authoring tools. - -Apple uses `clang` extensively for development and has upstreamed the necessary -functionality so that a vanilla clang can take advantage. It supports the use of `-F`, -`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when -building for macOS. - -Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the -FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several -other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These -do not build under Linux, so they have been patched to do so. The work here was used as -a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). - -In order to build a working toolchain, the following source packages are needed from -Apple: `cctools`, `dyld`, and `ld64`. - -These tools inject timestamps by default, which produce non-deterministic binaries. The -`ZERO_AR_DATE` environment variable is used to disable that. - -This version of `cctools` has been patched to use the current version of `clang`'s headers -and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. - -To complicate things further, all builds must target an Apple SDK. These SDKs are free to -download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download the [Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg). - -This file is several gigabytes in size, but only a single directory inside is needed: -``` -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk -``` - -Unfortunately, the usual Linux tools (7zip, hpmount, loopback mount) are incapable of -opening this file. To create a tarball suitable for Gitian input, there are two options: - -Using macOS, you can mount the DMG, and then create it with: -```shell -hdiutil attach Xcode_7.3.1.dmg -tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.11.sdk.tar.gz MacOSX10.11.sdk -``` - -Alternatively, you can use 7zip and SleuthKit to extract the files one by one. The script -[`extract-osx-sdk.sh`](./../contrib/macdeploy/extract-osx-sdk.sh) automates this. First -ensure the DMG file is in the current directory, and then run the script. You may wish to -delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the directory) when you've -confirmed the extraction succeeded. - -```shell -apt-get install p7zip-full sleuthkit -contrib/macdeploy/extract-osx-sdk.sh -rm -rf 5.hfs MacOSX10.11.sdk -``` - -The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are -created using these tools. The build process has been designed to avoid including the -SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely -redistributed. - -`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been -patched. A system `genisoimage` will work fine, but it will not be deterministic because -the file-order will change between invocations. The patch can be seen here: [theuni/osx-cross-depends](https://raw.githubusercontent.com/theuni/osx-cross-depends/master/patches/cdrtools/genisoimage.diff). -No effort was made to fix this cleanly, so it likely leaks memory badly. But it's only used for -a single invocation, so that's no real concern. - -`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the -`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its -maintainer has seemingly abandoned the project. It has been forked and is available -(with fixes) here: [theuni/libdmg-hfsplus](https://github.com/theuni/libdmg-hfsplus). - -The DMG tool has the ability to create DMGs from scratch as well, but this functionality is -broken. Only the compression feature is currently used. Ideally, the creation could be fixed -and `genisoimage` would no longer be necessary. - -Background images and other features can be added to DMG files by inserting a -`.DS_Store` before creation. This is generated by the script -`contrib/macdeploy/custom_dsstore.py`. - -As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in -order to satisfy the new Gatekeeper requirements. Because this private key cannot be -shared, we'll have to be a bit creative in order for the build process to remain somewhat -deterministic. Here's how it works: - -- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which - users may choose to bless and run. It also outputs an unsigned app structure in the form - of a tarball, which also contains all of the tools that have been previously (deterministically) - built in order to create a final DMG. -- The Apple keyholder uses this unsigned app to create a detached signature, using the - script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Gitian. It uses the - pre-built tools to recombine the pieces into a deterministic DMG. - diff --git a/doc/dependencies.md b/doc/dependencies.md index b739881a7a..51a2240107 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -10,9 +10,9 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Clang | | [3.3+](https://releases.llvm.org/download.html) (C++11 support) | | | | | Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | | +| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | -| HarfBuzz-NG | | | | | | +| HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | @@ -20,7 +20,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | +| Qt | [5.9.8](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | diff --git a/doc/descriptors.md b/doc/descriptors.md index a98f43737e..e31665b129 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -10,6 +10,13 @@ Supporting RPCs are: - `deriveaddresses` takes as input a descriptor and computes the corresponding addresses. - `listunspent` outputs a specialized descriptor for the reported unspent outputs. +- `getaddressinfo` outputs a descriptor for solvable addresses (since v0.18). +- `importmulti` takes as input descriptors to import into the wallet + (since v0.18). +- `generatetodescriptor` takes as input a descriptor and generates coins to it + (`regtest` only, since v0.19). +- `utxoupdatepsbt` takes as input descriptors to add information to the psbt + (since v0.19). This document describes the language. For the specifics on usage, see the RPC documentation for the functions mentioned above. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index b50f552e92..3ef35cfcfa 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -9,6 +9,7 @@ Developer Notes - [Coding Style (C++)](#coding-style-c) - [Coding Style (Python)](#coding-style-python) - [Coding Style (Doxygen-compatible comments)](#coding-style-doxygen-compatible-comments) + - [Generating Documentation](#generating-documentation) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) @@ -35,6 +36,9 @@ Developer Notes - [Source code organization](#source-code-organization) - [GUI](#gui) - [Subtrees](#subtrees) + - [Upgrading LevelDB](#upgrading-leveldb) + - [File Descriptor Counts](#file-descriptor-counts) + - [Consensus Compatibility](#consensus-compatibility) - [Scripted diffs](#scripted-diffs) - [Suggestions and examples](#suggestions-and-examples) - [Release notes](#release-notes) @@ -138,12 +142,17 @@ For example, to describe a function use: ```c++ /** - * ... text ... - * @param[in] arg1 A description - * @param[in] arg2 Another argument description - * @pre Precondition for function... + * ... Description ... + * + * @param[in] arg1 input description... + * @param[in] arg2 input description... + * @param[out] arg3 output description... + * @return Return cases... + * @throws Error type and cases... + * @pre Pre-condition for function... + * @post Post-condition for function... */ -bool function(int arg1, const char *arg2) +bool function(int arg1, const char *arg2, std::string& arg3) ``` A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. @@ -158,44 +167,73 @@ To describe a class, use the same construct above the class definition: * @see GetWarnings() */ class CAlert -{ ``` To describe a member or variable use: ```c++ -int var; //!< Detailed description after the member +//! Description before the member +int var; ``` or ```c++ -//! Description before the member -int var; +int var; //!< Description after the member ``` Also OK: ```c++ /// -/// ... text ... +/// ... Description ... /// bool function2(int arg1, const char *arg2) ``` -Not OK (used plenty in the current source, but not picked up): +Not picked up by Doxygen: ```c++ // -// ... text ... +// ... Description ... // ``` +Also not picked up by Doxygen: +```c++ +/* + * ... Description ... + */ +``` + A full list of comment syntaxes picked up by Doxygen can be found at http://www.doxygen.nl/manual/docblocks.html, but the above styles are favored. -Documentation can be generated with `make docs` and cleaned up with `make clean-docs`. The resulting files are located in `doc/doxygen/html`; open `index.html` to view the homepage. +Recommendations: -Before running `make docs`, you will need to install dependencies `doxygen` and `dot`. For example, on macOS via Homebrew: -``` -brew install graphviz doxygen -``` +- Avoiding duplicating type and input/output information in function + descriptions. + +- Use backticks (``) to refer to `argument` names in function and + parameter descriptions. + +- Backticks aren't required when referring to functions Doxygen already knows + about; it will build hyperlinks for these automatically. See + http://www.doxygen.nl/manual/autolink.html for complete info. + +- Avoid linking to external documentation; links can break. + +- Javadoc and all valid Doxygen comments are stripped from Doxygen source code + previews (`STRIP_CODE_COMMENTS = YES` in [Doxyfile.in](doc/Doxyfile.in)). If + you want a comment to be preserved, it must instead use `//` or `/* */`. + +### Generating Documentation + +The documentation can be generated with `make docs` and cleaned up with `make +clean-docs`. The resulting files are located in `doc/doxygen/html`; open +`index.html` in that directory to view the homepage. + +Before running `make docs`, you'll need to install these dependencies: + +Linux: `sudo apt install doxygen graphviz` + +MacOS: `brew install doxygen graphviz` Development tips and tricks --------------------------- @@ -919,7 +957,7 @@ introduce accidental changes. Some good examples of scripted-diff: - [scripted-diff: Rename InitInterfaces to NodeContext](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) -uses an elegant script to replace occurences of multiple terms in all source files. +uses an elegant script to replace occurrences of multiple terms in all source files. - [scripted-diff: Remove g_connman, g_banman globals](https://github.com/bitcoin/bitcoin/commit/301bd41a2e6765b185bd55f4c541f9e27aeea29d) replaces specific terms in a list of specific source files. @@ -1047,6 +1085,11 @@ A few guidelines for introducing and reviewing new RPC interfaces: new RPC is replacing a deprecated RPC, to avoid both RPCs confusingly showing up in the command list. +- Use *invalid* bech32 addresses for `RPCExamples` help documentation. + + - *Rationale*: Prevent accidental transactions by users and encourage the use + of bech32 addresses by default. + - Use the `UNIX_EPOCH_TIME` constant when describing UNIX epoch time or timestamps in the documentation. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 50e9251b8d..c34ca4cb59 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -7,11 +7,8 @@ describe how to use it with AFL and libFuzzer. ## Preparing fuzzing -AFL needs an input directory with examples, and an output directory where it -will place examples that it found. These can be anywhere in the file system, -we'll define environment variables to make it easy to reference them. - -libFuzzer will use the input directory as output directory. +The fuzzer needs some inputs to work on, but the inputs or seeds can be used +interchangeably between libFuzzer and AFL. Extract the example seeds (or other starting inputs) into the inputs directory before starting fuzzing. @@ -21,13 +18,19 @@ git clone https://github.com/bitcoin-core/qa-assets export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus ``` -Only for AFL: +AFL needs an input directory with examples, and an output directory where it +will place examples that it found. These can be anywhere in the file system, +we'll define environment variables to make it easy to reference them. + +So, only for AFL you need to configure the outputs path: ``` mkdir outputs export AFLOUT=$PWD/outputs ``` +libFuzzer will use the input directory as output directory. + ## AFL ### Building AFL @@ -41,6 +44,9 @@ make export AFLPATH=$PWD ``` +For macOS you may need to ignore x86 compilation checks when running `make`: +`AFL_NO_X86=1 make`. + ### Instrumentation To build Bitcoin Core using AFL instrumentation (this assumes that the @@ -48,9 +54,15 @@ To build Bitcoin Core using AFL instrumentation (this assumes that the ``` ./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ export AFL_HARDEN=1 -cd src/ make ``` + +If you are using clang you will need to substitute `afl-gcc` with `afl-clang` +and `afl-g++` with `afl-clang++`, so the first line above becomes: +``` +./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-clang CXX=${AFLPATH}/afl-clang++ +``` + We disable ccache because we don't want to pollute the ccache with instrumented objects, and similarly don't want to use non-instrumented cached objects linked in. @@ -60,25 +72,32 @@ The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting binary will be instrumented in such a way that the AFL features "persistent mode" and "deferred forkserver" can be used. See -https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. +https://github.com/google/AFL/tree/master/llvm_mode for details. ### Fuzzing To start the actual fuzzing use: ``` -export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target +export FUZZ_TARGET=bech32 # Pick a fuzz_target mkdir ${AFLOUT}/${FUZZ_TARGET} -$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET} +$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} ``` You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. +On macOS you may need to set `AFL_NO_FORKSRV=1` to get the target to run. +``` +export FUZZ_TARGET=bech32 # Pick a fuzz_target +mkdir ${AFLOUT}/${FUZZ_TARGET} +AFL_NO_FORKSRV=1 $AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- src/test/fuzz/${FUZZ_TARGET} +``` + ## libFuzzer -A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and libFuzzer is needed (all -found in the `compiler-rt` runtime libraries package). +A recent version of `clang`, the address/undefined sanitizers (ASan/UBSan) and +libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). To build all fuzz targets with libFuzzer, run @@ -87,11 +106,33 @@ To build all fuzz targets with libFuzzer, run make ``` -The fuzzer needs some inputs to work on, but the inputs or seeds can be used -interchangeably between libFuzzer and AFL. - See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer instrumented executable. -Alternatively run the script in `./test/fuzz/test_runner.py` and provide it -with the `${DIR_FUZZ_IN}` created earlier. +Alternatively, you can run the script through the fuzzing test harness (only +libFuzzer supported so far). You need to pass it the inputs directory and +the specific test target you want to run. + +``` +./test/fuzz/test_runner.py ${DIR_FUZZ_IN} bech32 +``` + +### macOS hints for libFuzzer + +The default clang/llvm version supplied by Apple on macOS does not include +fuzzing libraries, so macOS users will need to install a full version, for +example using `brew install llvm`. + +Should you run into problems with the address sanitizer, it is possible you +may need to run `./configure` with `--disable-asm` to avoid errors +with certain assembly code from Bitcoin Core's code. See [developer notes on sanitizers](https://github.com/bitcoin/bitcoin/blob/master/doc/developer-notes.md#sanitizers) +for more information. + +You may also need to take care of giving the correct path for clang and +clang++, like `CC=/path/to/clang CXX=/path/to/clang++` if the non-systems +clang does not come first in your path. + +Full configure that was tested on macOS Catalina with `brew` installed `llvm`: +``` +./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm +``` diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 8d8ccdfedc..b0faf0825a 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -41,7 +41,7 @@ threads take up 8MiB for the thread stack on a 64-bit system, and 4MiB in a By default, since glibc `2.10`, the C library will create up to two heap arenas per core. This is known to cause excessive memory usage in some scenarios. To avoid this make a script that sets `MALLOC_ARENA_MAX` before starting bitcoind: ```bash -#!/bin/bash +#!/usr/bin/env bash export MALLOC_ARENA_MAX=1 bitcoind ``` diff --git a/doc/release-notes-17578.md b/doc/release-notes-17578.md new file mode 100644 index 0000000000..664d17fd78 --- /dev/null +++ b/doc/release-notes-17578.md @@ -0,0 +1,13 @@ +Deprecated or removed RPCs +-------------------------- + +- RPC `getaddressinfo` changes: + + - the `label` field has been deprecated in favor of the `labels` field and + will be removed in 0.21. It can be re-enabled in the interim by launching + with `-deprecatedrpc=label`. + + - the `labels` behavior of returning an array of JSON objects containing name + and purpose key/value pairs has been deprecated in favor of an array of + label names and will be removed in 0.21. The previous behavior can be + re-enabled in the interim by launching with `-deprecatedrpc=labelspurpose`. diff --git a/share/genbuild.sh b/share/genbuild.sh index cd6214d5fa..197787d5e0 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 275078c88d..3befba3425 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -16,7 +16,7 @@ <key>CFBundlePackageType</key> <string>APPL</string> - <key>CFBundleGetInfoString</key> + <key>NSHumanReadableCopyright</key> <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@</string> <key>CFBundleShortVersionString</key> diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index 3cc0fa1b1d..4efc4bd6f5 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2018 The Bitcoin Core developers +# Copyright (c) 2012-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/share/setup.nsi.in b/share/setup.nsi.in index be482ae741..dd9ee54d6d 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -58,8 +58,8 @@ VIAddVersionKey ProductVersion "@PACKAGE_VERSION@" VIAddVersionKey CompanyName "${COMPANY}" VIAddVersionKey CompanyWebsite "${URL}" VIAddVersionKey FileVersion "@PACKAGE_VERSION@" -VIAddVersionKey FileDescription "" -VIAddVersionKey LegalCopyright "" +VIAddVersionKey FileDescription "Installer for @PACKAGE_NAME@" +VIAddVersionKey LegalCopyright "Copyright (C) 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@" InstallDirRegKey HKCU "${REGKEY}" Path ShowUninstDetails show diff --git a/src/Makefile.am b/src/Makefile.am index 27c87688b4..e58a89ca03 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,6 +210,7 @@ BITCOIN_CORE_H = \ txmempool.h \ ui_interface.h \ undo.h \ + util/asmap.h \ util/bip32.h \ util/bytevectorhash.h \ util/check.h \ @@ -510,6 +511,7 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ + util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ util/error.cpp \ @@ -554,12 +556,9 @@ if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif -# Libraries below may be listed more than once to resolve circular dependencies (see -# https://eli.thegreenplace.net/2013/07/09/library-order-in-static-linking#circular-dependency) bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ @@ -702,15 +701,20 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) +if TARGET_DARWIN + @echo "Checking macOS dynamic libraries..." + $(AM_V_at) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) +endif + if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." - $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) + $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif if EMBEDDED_LEVELDB diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index fad4873545..79db5cd7b4 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -10,6 +10,7 @@ QT_TS = \ qt/locale/bitcoin_de_DE.ts \ qt/locale/bitcoin_el.ts \ qt/locale/bitcoin_el_GR.ts \ + qt/locale/bitcoin_en.ts \ qt/locale/bitcoin_en_AU.ts \ qt/locale/bitcoin_en_GB.ts \ qt/locale/bitcoin_eo.ts \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 091ef50349..c76f30de8e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -7,6 +7,7 @@ FUZZ_TARGETS = \ test/fuzz/addr_info_deserialize \ test/fuzz/address_deserialize \ test/fuzz/addrman_deserialize \ + test/fuzz/asmap \ test/fuzz/banentry_deserialize \ test/fuzz/base_encode_decode \ test/fuzz/bech32 \ @@ -23,6 +24,7 @@ FUZZ_TARGETS = \ test/fuzz/blockundo_deserialize \ test/fuzz/bloomfilter_deserialize \ test/fuzz/coins_deserialize \ + test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ @@ -38,12 +40,12 @@ FUZZ_TARGETS = \ test/fuzz/out_point_deserialize \ test/fuzz/parse_hd_keypath \ test/fuzz/parse_iso8601 \ - test/fuzz/partial_merkle_tree_deserialize \ - test/fuzz/partially_signed_transaction_deserialize \ - test/fuzz/prefilled_transaction_deserialize \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ + test/fuzz/partial_merkle_tree_deserialize \ + test/fuzz/partially_signed_transaction_deserialize \ + test/fuzz/prefilled_transaction_deserialize \ test/fuzz/psbt \ test/fuzz/psbt_input_deserialize \ test/fuzz/psbt_output_deserialize \ @@ -53,6 +55,7 @@ FUZZ_TARGETS = \ test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/spanparsing \ + test/fuzz/strprintf \ test/fuzz/sub_net_deserialize \ test/fuzz/transaction \ test/fuzz/tx_in \ @@ -81,7 +84,8 @@ JSON_TEST_FILES = \ test/data/tx_invalid.json \ test/data/tx_valid.json -RAW_TEST_FILES = +RAW_TEST_FILES = \ + test/data/asmap.raw GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) @@ -185,6 +189,7 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_flush_tests.cpp \ test/versionbits_tests.cpp if ENABLE_PROPERTY_TESTS @@ -232,341 +237,360 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ -test_fuzz_block_SOURCES = $(FUZZ_SUITE) test/fuzz/block.cpp -test_fuzz_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 -test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 -test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 +test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addr_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addr_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 -test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 +test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 -test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_asmap_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_asmap_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_asmap_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_asmap_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_asmap_SOURCES = $(FUZZ_SUITE) test/fuzz/asmap.cpp -test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_bech32_SOURCES = $(FUZZ_SUITE) test/fuzz/bech32.cpp -test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_base_encode_decode_SOURCES = $(FUZZ_SUITE) test/fuzz/base_encode_decode.cpp test_fuzz_base_encode_decode_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_base_encode_decode_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_base_encode_decode_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_base_encode_decode_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_base_encode_decode_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_base_encode_decode_SOURCES = $(FUZZ_SUITE) test/fuzz/base_encode_decode.cpp -test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 -test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 -test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 -test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_descriptor_parse_SOURCES = $(FUZZ_SUITE) test/fuzz/descriptor_parse.cpp -test_fuzz_descriptor_parse_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_descriptor_parse_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_descriptor_parse_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_descriptor_parse_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bech32_SOURCES = $(FUZZ_SUITE) test/fuzz/bech32.cpp -test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 -test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_SOURCES = $(FUZZ_SUITE) test/fuzz/block.cpp -test_fuzz_parse_iso8601_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_iso8601.cpp -test_fuzz_parse_iso8601_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_iso8601_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 +test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_psbt_SOURCES = $(FUZZ_SUITE) test/fuzz/psbt.cpp -test_fuzz_psbt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_psbt_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_psbt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_psbt_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_file_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILE_INFO_DESERIALIZE=1 +test_fuzz_block_file_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_file_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_file_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_file_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_script_SOURCES = $(FUZZ_SUITE) test/fuzz/script.cpp -test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_filter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILTER_DESERIALIZE=1 +test_fuzz_block_filter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_filter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_filter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_filter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp -test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_header_and_short_txids_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE=1 +test_fuzz_block_header_and_short_txids_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_block_header_and_short_txids_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_header_and_short_txids_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 -test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 +test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_spanparsing_SOURCES = $(FUZZ_SUITE) test/fuzz/spanparsing.cpp -test_fuzz_spanparsing_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_spanparsing_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_spanparsing_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_spanparsing_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 +test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 -test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 +test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 -test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 +test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_hex_SOURCES = $(FUZZ_SUITE) test/fuzz/hex.cpp -test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 +test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 -test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 +test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 +test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_decode_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_decode_tx_SOURCES = $(FUZZ_SUITE) test/fuzz/decode_tx.cpp + +test_fuzz_descriptor_parse_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_descriptor_parse_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_descriptor_parse_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_descriptor_parse_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_descriptor_parse_SOURCES = $(FUZZ_SUITE) test/fuzz/descriptor_parse.cpp -test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_eval_script_SOURCES = $(FUZZ_SUITE) test/fuzz/eval_script.cpp test_fuzz_eval_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_eval_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_eval_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_eval_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_eval_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_eval_script_SOURCES = $(FUZZ_SUITE) test/fuzz/eval_script.cpp -test_fuzz_integer_SOURCES = $(FUZZ_SUITE) test/fuzz/integer.cpp -test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_integer_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 -test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 -test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 -test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_transaction_SOURCES = $(FUZZ_SUITE) test/fuzz/transaction.cpp -test_fuzz_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_addr_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 -test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_addr_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_block_file_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_block_file_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILE_INFO_DESERIALIZE=1 -test_fuzz_block_file_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_file_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_file_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_block_filter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_block_filter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILTER_DESERIALIZE=1 -test_fuzz_block_filter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_filter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_filter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_block_header_and_short_txids_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_block_header_and_short_txids_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE=1 -test_fuzz_block_header_and_short_txids_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_header_and_short_txids_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_fee_rate_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_fee_rate_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFEE_RATE_DESERIALIZE=1 test_fuzz_fee_rate_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fee_rate_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_flat_file_pos_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_flat_file_pos_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_hex_SOURCES = $(FUZZ_SUITE) test/fuzz/hex.cpp + +test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_integer_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_integer_SOURCES = $(FUZZ_SUITE) test/fuzz/integer.cpp + +test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 +test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_key_origin_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_key_origin_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DKEY_ORIGIN_INFO_DESERIALIZE=1 test_fuzz_key_origin_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_key_origin_info_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_merkle_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_merkle_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMERKLE_BLOCK_DESERIALIZE=1 test_fuzz_merkle_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_merkle_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_merkle_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_merkle_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_merkle_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 +test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 +test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_out_point_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_out_point_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DOUT_POINT_DESERIALIZE=1 test_fuzz_out_point_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_out_point_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_out_point_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_out_point_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_out_point_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_partially_signed_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_partially_signed_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIALLY_SIGNED_TRANSACTION_DESERIALIZE=1 -test_fuzz_partially_signed_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_hd_keypath_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_hd_keypath_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_hd_keypath_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_hd_keypath_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_hd_keypath_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_hd_keypath.cpp + +test_fuzz_parse_iso8601_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_iso8601_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_iso8601_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_iso8601.cpp + +test_fuzz_parse_numbers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_numbers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_numbers_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_numbers_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_numbers_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_numbers.cpp + +test_fuzz_parse_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_script_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_script.cpp + +test_fuzz_parse_univalue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_parse_univalue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_univalue_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_univalue.cpp -test_fuzz_partial_merkle_tree_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partial_merkle_tree_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_partially_signed_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIALLY_SIGNED_TRANSACTION_DESERIALIZE=1 +test_fuzz_partially_signed_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partially_signed_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_prefilled_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_prefilled_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPREFILLED_TRANSACTION_DESERIALIZE=1 test_fuzz_prefilled_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prefilled_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_psbt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_psbt_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_psbt_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_SOURCES = $(FUZZ_SUITE) test/fuzz/psbt.cpp -test_fuzz_psbt_input_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_psbt_input_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_INPUT_DESERIALIZE=1 test_fuzz_psbt_input_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_psbt_input_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_psbt_input_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_input_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_input_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_psbt_output_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_psbt_output_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_OUTPUT_DESERIALIZE=1 test_fuzz_psbt_output_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_psbt_output_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_psbt_output_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_psbt_output_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_output_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_pub_key_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_pub_key_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPUB_KEY_DESERIALIZE=1 test_fuzz_pub_key_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_pub_key_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_pub_key_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_pub_key_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_pub_key_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_SOURCES = $(FUZZ_SUITE) test/fuzz/script.cpp -test_fuzz_script_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_script_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSCRIPT_DESERIALIZE=1 test_fuzz_script_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_script_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_script_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp + +test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 +test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp + +test_fuzz_spanparsing_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_spanparsing_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_spanparsing_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_spanparsing_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_spanparsing_SOURCES = $(FUZZ_SUITE) test/fuzz/spanparsing.cpp + +test_fuzz_strprintf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_strprintf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_strprintf_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_strprintf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_strprintf_SOURCES = $(FUZZ_SUITE) test/fuzz/strprintf.cpp -test_fuzz_sub_net_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_sub_net_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSUB_NET_DESERIALIZE=1 test_fuzz_sub_net_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_sub_net_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_tx_in_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_tx_in_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTX_IN_DESERIALIZE=1 -test_fuzz_tx_in_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_tx_in_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_tx_in_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_transaction_SOURCES = $(FUZZ_SUITE) test/fuzz/transaction.cpp -test_fuzz_tx_in_SOURCES = $(FUZZ_SUITE) test/fuzz/tx_in.cpp test_fuzz_tx_in_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_tx_in_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_tx_in_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_tx_in_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_in_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_SOURCES = $(FUZZ_SUITE) test/fuzz/tx_in.cpp + +test_fuzz_tx_in_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTX_IN_DESERIALIZE=1 +test_fuzz_tx_in_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_tx_in_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_in_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_tx_out_SOURCES = $(FUZZ_SUITE) test/fuzz/tx_out.cpp test_fuzz_tx_out_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_tx_out_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_tx_out_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_tx_out_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_tx_out_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_out_SOURCES = $(FUZZ_SUITE) test/fuzz/tx_out.cpp -test_fuzz_parse_hd_keypath_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_hd_keypath.cpp -test_fuzz_parse_hd_keypath_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_hd_keypath_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_hd_keypath_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_parse_hd_keypath_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_parse_script_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_script.cpp -test_fuzz_parse_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_parse_script_LDADD = $(FUZZ_SUITE_LD_COMMON) - -test_fuzz_parse_numbers_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_numbers.cpp -test_fuzz_parse_numbers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_numbers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_numbers_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_parse_numbers_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 +test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp -test_fuzz_parse_univalue_SOURCES = $(FUZZ_SUITE) test/fuzz/parse_univalue.cpp -test_fuzz_parse_univalue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -test_fuzz_parse_univalue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 +test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp endif # ENABLE_FUZZ @@ -626,3 +650,12 @@ endif echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" + +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" diff --git a/src/addrdb.cpp b/src/addrdb.cpp index db936486b6..835c5d6c65 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/addrdb.h b/src/addrdb.h index ad85224d1f..c6d4307d69 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -49,15 +49,7 @@ public: banReason = ban_reason_in; } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(nCreateTime); - READWRITE(nBanUntil); - READWRITE(banReason); - } + SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); } void SetNull() { diff --git a/src/addrman.cpp b/src/addrman.cpp index 32676f8fa5..121ae4bf7e 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,20 +7,27 @@ #include <hash.h> #include <serialize.h> +#include <logging.h> -int CAddrInfo::GetTriedBucket(const uint256& nKey) const +int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i.\n", ToStringIP(), mapped_as, tried_bucket); + return tried_bucket; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const +int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const { - std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetCheapHash(); + std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - return hash2 % ADDRMAN_NEW_BUCKET_COUNT; + int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i.\n", ToStringIP(), mapped_as, new_bucket); + return new_bucket; } int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const @@ -153,7 +160,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -169,7 +176,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey); + int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -233,7 +240,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey); + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -301,7 +308,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; @@ -439,7 +446,7 @@ int CAddrMan::Check_() if (vvTried[n][i] != -1) { if (!setTried.count(vvTried[n][i])) return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n) + if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) return -17; if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) return -18; @@ -545,7 +552,7 @@ void CAddrMan::ResolveCollisions_() CAddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey); + int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -609,10 +616,33 @@ CAddrInfo CAddrMan::SelectTriedCollision_() CAddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); int id_old = vvTried[tried_bucket][tried_bucket_pos]; return mapInfo[id_old]; } + +std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) +{ + std::vector<bool> bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk.\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + fseek(filestr, 0, SEEK_SET); + char cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + return bits; +} diff --git a/src/addrman.h b/src/addrman.h index e54184ce35..5901611bee 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,11 +12,17 @@ #include <sync.h> #include <timedata.h> #include <util/system.h> +#include <clientversion.h> #include <map> #include <set> #include <stdint.h> #include <vector> +#include <iostream> +#include <streams.h> +#include <fs.h> +#include <hash.h> + /** * Extended statistics about a CAddress @@ -53,14 +59,10 @@ private: public: - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CAddress, *this); - READWRITE(source); - READWRITE(nLastSuccess); - READWRITE(nAttempts); + SERIALIZE_METHODS(CAddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); } CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) @@ -72,15 +74,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey) const; + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src) const; + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey) const + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const { - return GetNewBucket(nKey, source); + return GetNewBucket(nKey, source, asmap); } //! Calculate in which position of a bucket to store this entry. @@ -174,9 +176,10 @@ static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes */ class CAddrMan { +friend class CAddrManTest; protected: //! critical section to protect the inner data structures - mutable CCriticalSection cs; + mutable RecursiveMutex cs; private: //! last used nId @@ -268,9 +271,29 @@ protected: void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); public: + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + std::vector<bool> m_asmap; + + // Read asmap from provided binary file + static std::vector<bool> DecodeAsmap(fs::path path); + + /** * serialized format: - * * version byte (currently 1) + * * version byte (1 for pre-asmap files, 2 for files including asmap version) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -294,7 +317,7 @@ public: * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports * changes to the ADDRMAN_ parameters without breaking the on-disk structure. * - * We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has + * We don't use SERIALIZE_METHODS since the serialization and deserialization code has * very little in common. */ template<typename Stream> @@ -302,7 +325,7 @@ public: { LOCK(cs); - unsigned char nVersion = 1; + unsigned char nVersion = 2; s << nVersion; s << ((unsigned char)32); s << nKey; @@ -345,6 +368,13 @@ public: } } } + // Store asmap version after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_version; + if (m_asmap.size() != 0) { + asmap_version = SerializeHash(m_asmap); + } + s << asmap_version; } template<typename Stream> @@ -353,7 +383,6 @@ public: LOCK(cs); Clear(); - unsigned char nVersion; s >> nVersion; unsigned char nKeySize; @@ -383,16 +412,6 @@ public: mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); - if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) { - // In case the new table data cannot be used (nVersion unknown, or bucket count wrong), - // immediately try to give them a reference based on their primary source address. - int nUBucket = info.GetNewBucket(nKey); - int nUBucketPos = info.GetBucketPosition(nKey, true, nUBucket); - if (vvNew[nUBucket][nUBucketPos] == -1) { - vvNew[nUBucket][nUBucketPos] = n; - info.nRefCount++; - } - } } nIdCount = nNew; @@ -401,7 +420,7 @@ public: for (int n = 0; n < nTried; n++) { CAddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (vvTried[nKBucket][nKBucketPos] == -1) { info.nRandomPos = vRandom.size(); @@ -417,7 +436,9 @@ public: } nTried -= nLost; - // Deserialize positions in the new table (if possible). + // Store positions in the new table buckets to apply later (if possible). + std::map<int, int> entryToBucket; // Represents which entry belonged to which bucket when serializing + for (int bucket = 0; bucket < nUBuckets; bucket++) { int nSize = 0; s >> nSize; @@ -425,12 +446,38 @@ public: int nIndex = 0; s >> nIndex; if (nIndex >= 0 && nIndex < nNew) { - CAddrInfo &info = mapInfo[nIndex]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 1 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS) { - info.nRefCount++; - vvNew[bucket][nUBucketPos] = nIndex; - } + entryToBucket[nIndex] = bucket; + } + } + } + + uint256 supplied_asmap_version; + if (m_asmap.size() != 0) { + supplied_asmap_version = SerializeHash(m_asmap); + } + uint256 serialized_asmap_version; + if (nVersion > 1) { + s >> serialized_asmap_version; + } + + for (int n = 0; n < nNew; n++) { + CAddrInfo &info = mapInfo[n]; + int bucket = entryToBucket[n]; + int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; + } else { + // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + bucket = info.GetNewBucket(nKey, m_asmap); + nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][nUBucketPos] == -1) { + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; } } } diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 1111f27771..0bebb0cf54 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 171135b01f..a0a0429c2a 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/banman.cpp b/src/banman.cpp index 37fca7dd82..9cc584f0e4 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/banman.h b/src/banman.h index 7943f666e8..8984874914 100644 --- a/src/banman.h +++ b/src/banman.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BANMAN_H @@ -59,7 +59,7 @@ private: //!clean unused entries (if bantime has expired) void SweepBanned(); - CCriticalSection m_cs_banned; + RecursiveMutex m_cs_banned; banmap_t m_banned GUARDED_BY(m_cs_banned); bool m_is_dirty GUARDED_BY(m_cs_banned); CClientUIInterface* m_client_interface = nullptr; diff --git a/src/base58.cpp b/src/base58.cpp index 17d3f86ba8..6a9e21ffc2 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/base58.h b/src/base58.h index 90eded4992..042ad671d3 100644 --- a/src/base58.h +++ b/src/base58.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index d8c9b2e01a..0690483d50 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 80f13eeb3b..f2fc3999fe 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index cc159eb191..5cf7e43f4b 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -15,6 +15,9 @@ #include <numeric> #include <regex> +const RegTestingSetup* g_testing_setup = nullptr; +const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; + void benchmark::ConsolePrinter::header() { std::cout << "# Benchmark, evals, iterations, total, min, max, median" << std::endl; @@ -113,6 +116,8 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double for (const auto& p : benchmarks()) { RegTestingSetup test{}; + assert(g_testing_setup == nullptr); + g_testing_setup = &test; { LOCK(cs_main); assert(::ChainActive().Height() == 0); @@ -121,6 +126,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double } if (!std::regex_match(p.first, baseMatch, reFilter)) { + g_testing_setup = nullptr; continue; } @@ -133,6 +139,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double p.second.func(state); } printer.result(state); + g_testing_setup = nullptr; } printer.footer(); diff --git a/src/bench/bench.h b/src/bench/bench.h index 3a8c487b9a..6b7a0f76d1 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,9 @@ #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> +struct RegTestingSetup; +extern const RegTestingSetup* g_testing_setup; //!< A pointer to the current testing setup + // Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark // framework (see https://github.com/google/benchmark) // Why not use the Google Benchmark framework? Because adding Yet Another Dependency diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 184367e1e5..a113a73828 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -6,6 +6,7 @@ #include <consensus/validation.h> #include <crypto/sha256.h> #include <test/util/mining.h> +#include <test/util/setup_common.h> #include <test/util/wallet.h> #include <txmempool.h> #include <validation.h> @@ -29,7 +30,7 @@ static void AssembleBlock(benchmark::State& state) std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(SCRIPT_PUB)); + tx.vin.push_back(MineBlock(g_testing_setup->m_node, SCRIPT_PUB)); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, SCRIPT_PUB); if (NUM_BLOCKS - b >= COINBASE_MATURITY) @@ -46,7 +47,7 @@ static void AssembleBlock(benchmark::State& state) } while (state.KeepRunning()) { - PrepareBlock(SCRIPT_PUB); + PrepareBlock(g_testing_setup->m_node, SCRIPT_PUB); } } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 39cab092cf..c313029ea8 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index edf43bd4dc..55786126b3 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 000a0259bb..f5f96a0136 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 29a145bfe6..d6d5e67c5b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,8 @@ static void CoinSelection(benchmark::State& state) { NodeContext node; auto chain = interfaces::MakeChain(node); - const CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet.SetupLegacyScriptPubKeyMan(); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -64,7 +65,7 @@ static void CoinSelection(benchmark::State& state) typedef std::set<CInputCoin> CoinSet; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static const CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); +static CWallet testWallet(testChain.get(), WalletLocation(), WalletDatabase::CreateDummy()); std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp @@ -93,6 +94,7 @@ static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) static void BnBExhaustion(benchmark::State& state) { // Setup + testWallet.SetupLegacyScriptPubKeyMan(); std::vector<OutputGroup> utxo_pool; CoinSet selection; CAmount value_ret = 0; diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 674753c191..a9d4d78888 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index 3595249559..60a4fbf0ba 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index 0d9b123400..5d943810df 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index a2a21c673b..1c9c106666 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index cffdb388f8..6cdb4ff0a7 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 1c025e29d3..31e166cc27 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index f39dcc0b71..62568a9da5 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -7,6 +7,7 @@ #include <node/context.h> #include <optional.h> #include <test/util/mining.h> +#include <test/util/setup_common.h> #include <test/util/wallet.h> #include <validationinterface.h> #include <wallet/wallet.h> @@ -19,6 +20,7 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); CWallet wallet{chain.get(), WalletLocation(), WalletDatabase::CreateMock()}; { + wallet.SetupLegacyScriptPubKeyMan(); bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); wallet.handleNotifications(); @@ -29,8 +31,8 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { - generatetoaddress(address_mine.get_value_or(ADDRESS_WATCHONLY)); - generatetoaddress(ADDRESS_WATCHONLY); + generatetoaddress(g_testing_setup->m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); + generatetoaddress(g_testing_setup->m_node, ADDRESS_WATCHONLY); } SyncWithValidationInterfaceQueue(); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 1035e730b8..c085095a2b 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index c7af7e0fc8..735f55fba7 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index a2013edc05..7f1a4a114b 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -41,7 +41,7 @@ static bool WalletAppInit(int argc, char* argv[]) } if (argc < 2 || HelpRequested(gArgs)) { std::string usage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n\n" + - "bitcoin-wallet is an offline tool for creating and interacting with Bitcoin Core wallet files.\n" + + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index bf13297582..263d863cfa 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockencodings.h b/src/blockencodings.h index 18a6e35f31..55ed8989bb 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 5ad22d46eb..7aff3be6e7 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockfilter.h b/src/blockfilter.h index 828204b875..ff8744b217 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bloom.cpp b/src/bloom.cpp index a061925089..bd6069b31f 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bloom.h b/src/bloom.h index c3f64ba4bc..68e76a0258 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chain.cpp b/src/chain.cpp index 5520d8149a..c09113a866 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chain.h b/src/chain.h index 321bc95dbc..48bcb8bfdd 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -48,17 +48,15 @@ public: uint64_t nTimeFirst; //!< earliest time of block in file uint64_t nTimeLast; //!< latest time of block in file - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nBlocks)); - READWRITE(VARINT(nSize)); - READWRITE(VARINT(nUndoSize)); - READWRITE(VARINT(nHeightFirst)); - READWRITE(VARINT(nHeightLast)); - READWRITE(VARINT(nTimeFirst)); - READWRITE(VARINT(nTimeLast)); + SERIALIZE_METHODS(CBlockFileInfo, obj) + { + READWRITE(VARINT(obj.nBlocks)); + READWRITE(VARINT(obj.nSize)); + READWRITE(VARINT(obj.nUndoSize)); + READWRITE(VARINT(obj.nHeightFirst)); + READWRITE(VARINT(obj.nHeightLast)); + READWRITE(VARINT(obj.nTimeFirst)); + READWRITE(VARINT(obj.nTimeLast)); } void SetNull() { @@ -332,31 +330,25 @@ public: hashPrev = (pprev ? pprev->GetBlockHash() : uint256()); } - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CDiskBlockIndex, obj) + { int _nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); - - READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nStatus)); - READWRITE(VARINT(nTx)); - if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - if (nStatus & BLOCK_HAVE_DATA) - READWRITE(VARINT(nDataPos)); - if (nStatus & BLOCK_HAVE_UNDO) - READWRITE(VARINT(nUndoPos)); + if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); + + READWRITE(VARINT(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(obj.nStatus)); + READWRITE(VARINT(obj.nTx)); + if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED)); + if (obj.nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(obj.nDataPos)); + if (obj.nStatus & BLOCK_HAVE_UNDO) READWRITE(VARINT(obj.nUndoPos)); // block header - READWRITE(this->nVersion); - READWRITE(hashPrev); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); + READWRITE(obj.nVersion); + READWRITE(obj.hashPrev); + READWRITE(obj.hashMerkleRoot); + READWRITE(obj.nTime); + READWRITE(obj.nBits); + READWRITE(obj.nNonce); } uint256 GetBlockHash() const diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 0a1cb858ef..31592b0f0a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.h b/src/chainparams.h index 6be066806b..63398e587e 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index 4bb66c8d8b..894b8553c4 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 69fe2438f3..3c139931ea 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/checkqueue.h b/src/checkqueue.h index 978e23a7c4..9dab5a09ac 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -90,8 +90,7 @@ private: nTotal--; bool fRet = fAllOk; // reset the status for new work later - if (fMaster) - fAllOk = true; + fAllOk = true; // return the current status return fRet; } diff --git a/src/coins.cpp b/src/coins.cpp index 6b85edd01a..b71362c6a0 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/coins.h b/src/coins.h index d8135e0d9a..e71c8a47bc 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -61,7 +61,7 @@ public: assert(!IsSpent()); uint32_t code = nHeight * 2 + fCoinBase; ::Serialize(s, VARINT(code)); - ::Serialize(s, CTxOutCompressor(REF(out))); + ::Serialize(s, Using<TxOutCompression>(out)); } template<typename Stream> @@ -70,7 +70,7 @@ public: ::Unserialize(s, VARINT(code)); nHeight = code >> 1; fCoinBase = code & 1; - ::Unserialize(s, CTxOutCompressor(out)); + ::Unserialize(s, Using<TxOutCompression>(out)); } bool IsSpent() const { diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index c254fe7cbf..27ef1a18df 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/stdin.cpp b/src/compat/stdin.cpp index 98d406cca8..0fc4e0fcf2 100644 --- a/src/compat/stdin.cpp +++ b/src/compat/stdin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compressor.cpp b/src/compressor.cpp index a7f45b5c1e..a70306d320 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compressor.h b/src/compressor.h index c1eda503c8..7bb60d311e 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -11,10 +11,6 @@ #include <serialize.h> #include <span.h> -class CKeyID; -class CPubKey; -class CScriptID; - bool CompressScript(const CScript& script, std::vector<unsigned char> &out); unsigned int GetSpecialScriptSize(unsigned int nSize); bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); @@ -33,9 +29,8 @@ uint64_t DecompressAmount(uint64_t nAmount); * Other scripts up to 121 bytes require 1 byte + script length. Above * that, scripts up to 16505 bytes require 2 bytes + script length. */ -class CScriptCompressor +struct ScriptCompression { -private: /** * make this static for now (there are only 6 special scripts defined) * this can potentially be extended together with a new nVersion for @@ -44,12 +39,8 @@ private: */ static const unsigned int nSpecialScripts = 6; - CScript &script; -public: - explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { } - template<typename Stream> - void Serialize(Stream &s) const { + void Ser(Stream &s, const CScript& script) { std::vector<unsigned char> compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); @@ -61,7 +52,7 @@ public: } template<typename Stream> - void Unserialize(Stream &s) { + void Unser(Stream &s, CScript& script) { unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { @@ -82,30 +73,24 @@ public: } }; -/** wrapper for CTxOut that provides a more compact serialization */ -class CTxOutCompressor +struct AmountCompression { -private: - CTxOut &txout; - -public: - explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { } - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - if (!ser_action.ForRead()) { - uint64_t nVal = CompressAmount(txout.nValue); - READWRITE(VARINT(nVal)); - } else { - uint64_t nVal = 0; - READWRITE(VARINT(nVal)); - txout.nValue = DecompressAmount(nVal); - } - CScriptCompressor cscript(REF(txout.scriptPubKey)); - READWRITE(cscript); + template<typename Stream, typename I> void Ser(Stream& s, I val) + { + s << VARINT(CompressAmount(val)); + } + template<typename Stream, typename I> void Unser(Stream& s, I& val) + { + uint64_t v; + s >> VARINT(v); + val = DecompressAmount(v); } }; +/** wrapper for CTxOut that provides a more compact serialization */ +struct TxOutCompression +{ + FORMATTER_METHODS(CTxOut, obj) { READWRITE(Using<AmountCompression>(obj.nValue), Using<ScriptCompression>(obj.scriptPubKey)); } +}; + #endif // BITCOIN_COMPRESSOR_H diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index f87612edef..843985e54c 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index f28f76bd34..4ae5a5b897 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/params.h b/src/consensus/params.h index e191fd6d26..61b1fbc2e5 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index 88bb12c713..bb8cd10c63 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_check.h b/src/consensus/tx_check.h index b818a284f1..21f842408a 100644 --- a/src/consensus/tx_check.h +++ b/src/consensus/tx_check.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 31bdabea28..81245e3e11 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index b6599f2878..ffcaf3cab1 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/validation.h b/src/consensus/validation.h index 3401eb64ca..8a3abb31f4 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_io.h b/src/core_io.h index 19fb7b29f6..80ec80cd50 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_read.cpp b/src/core_read.cpp index a3c9cf0159..9a65b02585 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_write.cpp b/src/core_write.cpp index 7ce2a49836..cb1fc214eb 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 6ed9088434..2afcbd1629 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/aes.h b/src/crypto/aes.h index e06c8de272..3a0011bee4 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 42a17f02ff..f3ff4268ee 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index 5a4674f4a8..69fbbe9fa5 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/hkdf_sha256_32.cpp b/src/crypto/hkdf_sha256_32.cpp index e684eced37..81f3c1349a 100644 --- a/src/crypto/hkdf_sha256_32.cpp +++ b/src/crypto/hkdf_sha256_32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/ripemd160.cpp b/src/crypto/ripemd160.cpp index edee06cc34..29a4ad906f 100644 --- a/src/crypto/ripemd160.cpp +++ b/src/crypto/ripemd160.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha1.cpp b/src/crypto/sha1.cpp index 3dcdcb186e..1fb9bb2b72 100644 --- a/src/crypto/sha1.cpp +++ b/src/crypto/sha1.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index dda7e5230f..e35d526d35 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha256_avx2.cpp b/src/crypto/sha256_avx2.cpp index 90a72516a4..624bdb42e4 100644 --- a/src/crypto/sha256_avx2.cpp +++ b/src/crypto/sha256_avx2.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifdef ENABLE_AVX2 #include <stdint.h> diff --git a/src/crypto/sha256_shani.cpp b/src/crypto/sha256_shani.cpp index 7ea0c34796..92f67710fb 100644 --- a/src/crypto/sha256_shani.cpp +++ b/src/crypto/sha256_shani.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/crypto/sha256_sse41.cpp b/src/crypto/sha256_sse41.cpp index fc79f46f7f..4eaf7d7b18 100644 --- a/src/crypto/sha256_sse41.cpp +++ b/src/crypto/sha256_sse41.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifdef ENABLE_SSE41 #include <stdint.h> diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 4e6aa363f7..85a7bbcb53 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha512.h b/src/crypto/sha512.h index fc7dd1b87e..21ca930c75 100644 --- a/src/crypto/sha512.h +++ b/src/crypto/sha512.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 34896f7ab2..d7694108f5 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 061c9b6bca..116d7d8679 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 38b5b0efc4..a5582e3b2c 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,8 @@ enum class WalletCreationStatus; namespace interfaces { class Chain; +class Handler; +class Wallet; } class DummyWalletInit : public WalletInitInterface { @@ -80,9 +82,13 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& throw std::logic_error("Wallet function called in non-wallet build."); } -namespace interfaces { +using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; +std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) +{ + throw std::logic_error("Wallet function called in non-wallet build."); +} -class Wallet; +namespace interfaces { std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { diff --git a/src/fs.cpp b/src/fs.cpp index 73fb3b606e..066c6c10d3 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <fs.h> #ifndef WIN32 @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 77e09bd5c7..ff75789223 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httprpc.h b/src/httprpc.h index 91c2ec0c9d..99e4d59b8a 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index d9c7113323..0e13b85806 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -172,7 +172,7 @@ static bool InitHTTPAllowList() rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; - LookupSubNet(strAllow.c_str(), subnet); + LookupSubNet(strAllow, subnet); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), @@ -324,7 +324,7 @@ static bool HTTPBindAddresses(struct evhttp* http) evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { CNetAddr addr; - if (i->first.empty() || (LookupHost(i->first.c_str(), addr, false) && addr.IsBindAny())) { + if (i->first.empty() || (LookupHost(i->first, addr, false) && addr.IsBindAny())) { LogPrintf("WARNING: the RPC server is not safe to expose to untrusted networks such as the public internet\n"); } boundSockets.push_back(bind_handle); diff --git a/src/httpserver.h b/src/httpserver.h index bc72fc8512..46820e6aee 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.cpp b/src/index/base.cpp index bcc8e2ce7c..dcb8e99fc1 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.h b/src/index/base.h index f95eeb8197..d0088d9c9a 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 62db38f894..5bbe6ad1df 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index ab5e3b9342..90d2624c7f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -52,7 +52,11 @@ #include <util/threadnames.h> #include <util/translation.h> #include <util/validation.h> +#include <util/asmap.h> #include <validation.h> +#include <hash.h> + + #include <validationinterface.h> #include <walletinitinterface.h> @@ -98,6 +102,8 @@ static constexpr int DUMP_BANS_INTERVAL = 60 * 15; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; + /** * The PID file facilities. */ @@ -172,7 +178,7 @@ void Interrupt(NodeContext& node) void Shutdown(NodeContext& node) { LogPrintf("%s: In progress...\n", __func__); - static CCriticalSection cs_Shutdown; + static RecursiveMutex cs_Shutdown; TRY_LOCK(cs_Shutdown, lockShutdown); if (!lockShutdown) return; @@ -197,8 +203,6 @@ void Shutdown(NodeContext& node) // using the other before destroying them. if (node.peer_logic) UnregisterValidationInterface(node.peer_logic.get()); if (node.connman) node.connman->Stop(); - if (g_txindex) g_txindex->Stop(); - ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); StopTorControl(); @@ -212,8 +216,6 @@ void Shutdown(NodeContext& node) node.peer_logic.reset(); node.connman.reset(); node.banman.reset(); - g_txindex.reset(); - DestroyAllBlockFilterIndexes(); if (::mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(::mempool); @@ -246,6 +248,14 @@ void Shutdown(NodeContext& node) // CValidationInterface callbacks, flush them... GetMainSignals().FlushBackgroundCallbacks(); + // Stop and delete all indexes only after flushing background callbacks. + if (g_txindex) { + g_txindex->Stop(); + g_txindex.reset(); + } + ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); + DestroyAllBlockFilterIndexes(); + // Any future callbacks will be dropped. This should absolutely be safe - if // missing a callback results in an unrecoverable situation, unclean shutdown // would too. The only reason to do the above flushes is to let the wallet catch @@ -426,7 +436,8 @@ void SetupServerArgs() gArgs.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); gArgs.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); + gArgs.AddArg("-asmap=<file>", "Specify asn mapping used for bucketing of the peers. Path should be relative to the -datadir path.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -517,7 +528,7 @@ void SetupServerArgs() gArgs.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted inbound peers with default permissions. This will relay transactions even if the transactions were already in the mempool or violate local relay policy. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. The will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. This will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-blockmaxweight=<n>", strprintf("Set maximum BIP141 block weight (default: %d)", DEFAULT_BLOCK_MAX_WEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); @@ -526,15 +537,15 @@ void SetupServerArgs() gArgs.AddArg("-rest", strprintf("Accept public REST requests (default: %u)", DEFAULT_REST_ENABLE), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); gArgs.AddArg("-rpcallowip=<ip>", "Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcauth=<userpw>", "Username and HMAC-SHA-256 hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::RPC); + gArgs.AddArg("-rpcauth=<userpw>", "Username and HMAC-SHA-256 hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); + gArgs.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY | ArgsManager::SENSITIVE, OptionsCategory::RPC); gArgs.AddArg("-rpccookiefile=<loc>", "Location of the auth cookie. Relative paths will be prefixed by a net-specific datadir location. (default: data dir)", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); + gArgs.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); gArgs.AddArg("-rpcport=<port>", strprintf("Listen for JSON-RPC connections on <port> (default: %u, testnet: %u, regtest: %u)", defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort(), regtestBaseParams->RPCPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::RPC); gArgs.AddArg("-rpcserialversion", strprintf("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)", DEFAULT_RPC_SERIALIZE_VERSION), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); gArgs.AddArg("-rpcservertimeout=<n>", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); gArgs.AddArg("-rpcthreads=<n>", strprintf("Set the number of threads to service RPC calls (default: %d)", DEFAULT_HTTP_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); - gArgs.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); + gArgs.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); gArgs.AddArg("-rpcwhitelist=<whitelist>", "Set a whitelist to filter incoming RPC calls for a specific user. The field <whitelist> comes in the format: <USERNAME>:<rpc 1>,<rpc 2>,...,<rpc n>. If multiple whitelists are set for a given user, they are set-intersected. See -rpcwhitelistdefault documentation for information on default whitelist behavior.", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); gArgs.AddArg("-rpcwhitelistdefault", "Sets default behavior for rpc whitelisting. Unless rpcwhitelistdefault is set to 0, if any -rpcwhitelist is set, the rpc server acts as if all rpc users are subject to empty-unless-otherwise-specified whitelists. If rpcwhitelistdefault is set to 1 and no -rpcwhitelist is set, rpc server acts as if all rpc users are subject to empty whitelists.", ArgsManager::ALLOW_BOOL, OptionsCategory::RPC); gArgs.AddArg("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); @@ -877,8 +888,8 @@ bool AppInitBasicSetup() _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); #endif #ifdef WIN32 - // Enable Data Execution Prevention (DEP) - SetProcessDEPPolicy(PROCESS_DEP_ENABLE); + // Enable heap terminate-on-corruption + HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif if (!SetupNetworking()) @@ -1219,6 +1230,9 @@ bool AppInitMain(NodeContext& node) LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); } + // Log the config arguments to debug.log + gArgs.LogArgs(); + LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. @@ -1354,7 +1368,7 @@ bool AppInitMain(NodeContext& node) SetReachable(NET_ONION, false); if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; - if (!Lookup(proxyArg.c_str(), proxyAddr, 9050, fNameLookup)) { + if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'").translated, proxyArg)); } @@ -1378,7 +1392,7 @@ bool AppInitMain(NodeContext& node) SetReachable(NET_ONION, false); } else { CService onionProxy; - if (!Lookup(onionArg.c_str(), onionProxy, 9050, fNameLookup)) { + if (!Lookup(onionArg, onionProxy, 9050, fNameLookup)) { return InitError(strprintf(_("Invalid -onion address or hostname: '%s'").translated, onionArg)); } proxyType addrOnion = proxyType(onionProxy, proxyRandomize); @@ -1396,7 +1410,7 @@ bool AppInitMain(NodeContext& node) for (const std::string& strAddr : gArgs.GetArgs("-externalip")) { CService addrLocal; - if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) + if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); else return InitError(ResolveErrMsg("externalip", strAddr)); @@ -1776,7 +1790,7 @@ bool AppInitMain(NodeContext& node) for (const std::string& strBind : gArgs.GetArgs("-bind")) { CService addrBind; - if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) { + if (!Lookup(strBind, addrBind, GetListenPort(), false)) { return InitError(ResolveErrMsg("bind", strBind)); } connOptions.vBinds.push_back(addrBind); @@ -1809,6 +1823,25 @@ bool AppInitMain(NodeContext& node) return false; } + // Read asmap file if configured + if (gArgs.IsArgSet("-asmap")) { + std::string asmap_file = gArgs.GetArg("-asmap", ""); + if (asmap_file.empty()) { + asmap_file = DEFAULT_ASMAP_FILENAME; + } + const fs::path asmap_path = GetDataDir() / asmap_file; + std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not find or parse specified asmap: '%s'").translated, asmap_path)); + return false; + } + const uint256 asmap_version = SerializeHash(asmap); + node.connman->SetAsmap(std::move(asmap)); + LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing.\n"); + } + // ********************************************************* Step 13: finished SetRPCWarmupFinished(); diff --git a/src/init.h b/src/init.h index ca52dadf08..f74ae5a47a 100644 --- a/src/init.h +++ b/src/init.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index ac640aa35a..643bb58d56 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,7 +38,7 @@ namespace interfaces { namespace { -class LockImpl : public Chain::Lock, public UniqueLock<CCriticalSection> +class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> { Optional<int> getHeight() override { @@ -338,7 +338,6 @@ public: void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } void initWarning(const std::string& message) override { InitWarning(message); } void initError(const std::string& message) override { InitError(message); } - void loadWallet(std::unique_ptr<Wallet> wallet) override { ::uiInterface.LoadWallet(wallet); } void showProgress(const std::string& title, int progress, bool resume_possible) override { ::uiInterface.ShowProgress(title, progress, resume_possible); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 349af152d5..7304f82749 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -43,7 +43,7 @@ class Wallet; //! asynchronously //! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). //! -//! * The initMessages() and loadWallet() methods which the wallet uses to send +//! * The initMessage() and showProgress() methods which the wallet uses to send //! notifications to the GUI should go away when GUI and wallet can directly //! communicate with each other without going through the node //! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). @@ -209,9 +209,6 @@ public: //! Send init error. virtual void initError(const std::string& message) = 0; - //! Send wallet load notification to the GUI. - virtual void loadWallet(std::unique_ptr<Wallet> wallet) = 0; - //! Send progress indicator. virtual void showProgress(const std::string& title, int progress, bool resume_possible) = 0; diff --git a/src/interfaces/handler.cpp b/src/interfaces/handler.cpp index 92601fc4e9..95035c1b54 100644 --- a/src/interfaces/handler.cpp +++ b/src/interfaces/handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,6 +22,15 @@ public: boost::signals2::scoped_connection m_connection; }; +class CleanupHandler : public Handler +{ +public: + explicit CleanupHandler(std::function<void()> cleanup) : m_cleanup(std::move(cleanup)) {} + ~CleanupHandler() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } + void disconnect() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } + std::function<void()> m_cleanup; +}; + } // namespace std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection) @@ -29,4 +38,9 @@ std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection) return MakeUnique<HandlerImpl>(std::move(connection)); } +std::unique_ptr<Handler> MakeHandler(std::function<void()> cleanup) +{ + return MakeUnique<CleanupHandler>(std::move(cleanup)); +} + } // namespace interfaces diff --git a/src/interfaces/handler.h b/src/interfaces/handler.h index c4c674cac5..fbac3c6b71 100644 --- a/src/interfaces/handler.h +++ b/src/interfaces/handler.h @@ -1,10 +1,11 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_INTERFACES_HANDLER_H #define BITCOIN_INTERFACES_HANDLER_H +#include <functional> #include <memory> namespace boost { @@ -30,6 +31,9 @@ public: //! Return handler wrapping a boost signal connection. std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection); +//! Return handler wrapping a cleanup function. +std::unique_ptr<Handler> MakeHandler(std::function<void()> cleanup); + } // namespace interfaces #endif // BITCOIN_INTERFACES_HANDLER_H diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 529fa793dd..8a64a9d26a 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -43,11 +43,10 @@ std::vector<fs::path> ListWalletDir(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::vector<std::string>& warnings); WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, std::string& error, std::vector<std::string>& warnings, std::shared_ptr<CWallet>& result); +std::unique_ptr<interfaces::Handler> HandleLoadWallet(interfaces::Node::LoadWalletFn load_wallet); namespace interfaces { -class Wallet; - namespace { class NodeImpl : public Node @@ -286,7 +285,7 @@ public: } std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override { - return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::unique_ptr<Wallet>& wallet) { fn(std::move(wallet)); })); + return HandleLoadWallet(std::move(fn)); } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 6bc4668beb..38aeb06324 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 94c6e8c7b7..baea71d0bb 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -119,7 +119,7 @@ public: } bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) override { - const SigningProvider* provider = m_wallet->GetSigningProvider(script); + std::unique_ptr<SigningProvider> provider = m_wallet->GetSigningProvider(script); if (provider) { return provider->GetPubKey(address, pub_key); } @@ -127,7 +127,7 @@ public: } bool getPrivKey(const CScript& script, const CKeyID& address, CKey& key) override { - const SigningProvider* provider = m_wallet->GetSigningProvider(script); + std::unique_ptr<SigningProvider> provider = m_wallet->GetSigningProvider(script); if (provider) { return provider->GetKey(address, key); } @@ -180,7 +180,6 @@ public: } return result; } - void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->GetLegacyScriptPubKeyMan()->LearnRelatedScripts(key, type); } bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { LOCK(m_wallet->cs_wallet); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 8d2b8a2eca..d4280e8091 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -108,10 +108,6 @@ public: //! Get wallet address list. virtual std::vector<WalletAddress> getAddresses() = 0; - //! Add scripts to key store so old so software versions opening the wallet - //! database can detect payments to newer address types. - virtual void learnRelatedScripts(const CPubKey& key, OutputType type) = 0; - //! Add dest data. virtual bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) = 0; diff --git a/src/key.cpp b/src/key.cpp index 10b6668aae..b6ed29e8e3 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/key_io.cpp b/src/key_io.cpp index af06db7343..d2f5be93f5 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/logging.cpp b/src/logging.cpp index 9f6b5ede12..6fd916b603 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -162,6 +162,7 @@ const CLogCategoryDesc LogCategories[] = {BCLog::COINDB, "coindb"}, {BCLog::QT, "qt"}, {BCLog::LEVELDB, "leveldb"}, + {BCLog::VALIDATION, "validation"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index a2caef51a8..b2fde1b9ea 100644 --- a/src/logging.h +++ b/src/logging.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -54,6 +54,7 @@ namespace BCLog { COINDB = (1 << 18), QT = (1 << 19), LEVELDB = (1 << 20), + VALIDATION = (1 << 21), ALL = ~(uint32_t)0, }; diff --git a/src/logging/timer.h b/src/logging/timer.h index 34dbb942c5..45bfc4aa65 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 052aebbc80..4ac6219886 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/miner.cpp b/src/miner.cpp index 2f6feb9e02..6f4e10b6ed 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -45,7 +45,9 @@ BlockAssembler::Options::Options() { nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; } -BlockAssembler::BlockAssembler(const CChainParams& params, const Options& options) : chainparams(params) +BlockAssembler::BlockAssembler(const CTxMemPool& mempool, const CChainParams& params, const Options& options) + : chainparams(params), + m_mempool(mempool) { blockMinFeeRate = options.blockMinFeeRate; // Limit weight to between 4K and MAX_BLOCK_WEIGHT-4K for sanity: @@ -67,7 +69,8 @@ static BlockAssembler::Options DefaultOptions() return options; } -BlockAssembler::BlockAssembler(const CChainParams& params) : BlockAssembler(params, DefaultOptions()) {} +BlockAssembler::BlockAssembler(const CTxMemPool& mempool, const CChainParams& params) + : BlockAssembler(mempool, params, DefaultOptions()) {} void BlockAssembler::resetBlock() { @@ -103,7 +106,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate->vTxFees.push_back(-1); // updated at end pblocktemplate->vTxSigOpsCost.push_back(-1); // updated at end - LOCK2(cs_main, mempool.cs); + LOCK2(cs_main, m_mempool.cs); CBlockIndex* pindexPrev = ::ChainActive().Tip(); assert(pindexPrev != nullptr); nHeight = pindexPrev->nHeight + 1; @@ -236,7 +239,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already int nDescendantsUpdated = 0; for (CTxMemPool::txiter it : alreadyAdded) { CTxMemPool::setEntries descendants; - mempool.CalculateDescendants(it, descendants); + m_mempool.CalculateDescendants(it, descendants); // Insert all descendants (not yet in block) into the modified set for (CTxMemPool::txiter desc : descendants) { if (alreadyAdded.count(desc)) @@ -268,7 +271,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already // cached size/sigops/fee values that are not actually correct. bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) { - assert (it != mempool.mapTx.end()); + assert(it != m_mempool.mapTx.end()); return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); } @@ -305,7 +308,7 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda // and modifying them for their already included ancestors UpdatePackagesForAdded(inBlock, mapModifiedTx); - CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); + CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = m_mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; // Limit the number of attempts to add transactions to the block when it is @@ -314,11 +317,10 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda const int64_t MAX_CONSECUTIVE_FAILURES = 1000; int64_t nConsecutiveFailed = 0; - while (mi != mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty()) - { + while (mi != m_mempool.mapTx.get<ancestor_score>().end() || !mapModifiedTx.empty()) { // First try to find a new transaction in mapTx to evaluate. - if (mi != mempool.mapTx.get<ancestor_score>().end() && - SkipMapTxEntry(mempool.mapTx.project<0>(mi), mapModifiedTx, failedTx)) { + if (mi != m_mempool.mapTx.get<ancestor_score>().end() && + SkipMapTxEntry(m_mempool.mapTx.project<0>(mi), mapModifiedTx, failedTx)) { ++mi; continue; } @@ -328,13 +330,13 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda bool fUsingModified = false; modtxscoreiter modit = mapModifiedTx.get<ancestor_score>().begin(); - if (mi == mempool.mapTx.get<ancestor_score>().end()) { + if (mi == m_mempool.mapTx.get<ancestor_score>().end()) { // We're out of entries in mapTx; use the entry from mapModifiedTx iter = modit->iter; fUsingModified = true; } else { // Try to compare the mapTx entry to the mapModifiedTx entry - iter = mempool.mapTx.project<0>(mi); + iter = m_mempool.mapTx.project<0>(mi); if (modit != mapModifiedTx.get<ancestor_score>().end() && CompareTxMemPoolEntryByAncestorFee()(*modit, CTxMemPoolModifiedEntry(iter))) { // The best entry in mapModifiedTx has higher score @@ -389,7 +391,7 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda CTxMemPool::setEntries ancestors; uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - mempool.CalculateMemPoolAncestors(*iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); + m_mempool.CalculateMemPoolAncestors(*iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); onlyUnconfirmed(ancestors); ancestors.insert(iter); diff --git a/src/miner.h b/src/miner.h index 7c4c455072..cc8fc31a9f 100644 --- a/src/miner.h +++ b/src/miner.h @@ -147,6 +147,7 @@ private: int nHeight; int64_t nLockTimeCutoff; const CChainParams& chainparams; + const CTxMemPool& m_mempool; public: struct Options { @@ -155,8 +156,8 @@ public: CFeeRate blockMinFeeRate; }; - explicit BlockAssembler(const CChainParams& params); - BlockAssembler(const CChainParams& params, const Options& options); + explicit BlockAssembler(const CTxMemPool& mempool, const CChainParams& params); + explicit BlockAssembler(const CTxMemPool& mempool, const CChainParams& params, const Options& options); /** Construct a new block template with coinbase to scriptPubKeyIn */ std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); @@ -175,7 +176,7 @@ private: /** Add transactions based on feerate including unconfirmed ancestors * Increments nPackagesSelected / nDescendantsUpdated with corresponding * statistics from the package selection (for logging statistics). */ - void addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + void addPackageTxs(int& nPackagesSelected, int& nDescendantsUpdated) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); // helper functions for addPackageTxs() /** Remove confirmed (inBlock) entries from given set */ @@ -189,13 +190,13 @@ private: bool TestPackageTransactions(const CTxMemPool::setEntries& package); /** Return true if given transaction from mapTx has already been evaluated, * or if the transaction's cached data in mapTx is incorrect. */ - bool SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + bool SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set& mapModifiedTx, CTxMemPool::setEntries& failedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); /** Sort the package in an order that is valid to appear in a block */ void SortForBlock(const CTxMemPool::setEntries& package, std::vector<CTxMemPool::txiter>& sortedEntries); /** Add descendants of given transactions to mapModifiedTx with ancestor * state updated assuming given transactions are inBlock. Returns number * of updated descendants. */ - int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set &mapModifiedTx) EXCLUSIVE_LOCKS_REQUIRED(mempool.cs); + int UpdatePackagesForAdded(const CTxMemPool::setEntries& alreadyAdded, indexed_modified_transaction_set& mapModifiedTx) EXCLUSIVE_LOCKS_REQUIRED(m_mempool.cs); }; /** Modify the extranonce in a block */ diff --git a/src/net.cpp b/src/net.cpp index 99dae88bab..18fe95e675 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -85,7 +85,7 @@ static const uint64_t RANDOMIZER_ID_LOCALHOSTNONCE = 0xd93e69e2bbfa5735ULL; // S bool fDiscover = true; bool fListen = true; bool g_relay_txes = !DEFAULT_BLOCKSONLY; -CCriticalSection cs_mapLocalHost; +RecursiveMutex cs_mapLocalHost; std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; std::string strSubVersion; @@ -410,7 +410,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (hSocket == INVALID_SOCKET) { return nullptr; } - connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, &proxyConnectionFailed); + connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, proxyConnectionFailed); } else { // no proxy needed (none set for target network) hSocket = CreateSocket(addrConnect); @@ -432,7 +432,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo std::string host; int port = default_port; SplitHostPort(std::string(pszDest), port, host); - connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, nullptr); + bool proxyConnectionFailed; + connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, proxyConnectionFailed); } if (!connected) { CloseSocket(hSocket); @@ -497,12 +498,13 @@ void CNode::SetAddrLocal(const CService& addrLocalIn) { #undef X #define X(name) stats.name = name -void CNode::copyStats(CNodeStats &stats) +void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) { stats.nodeid = this->GetId(); X(nServices); X(addr); X(addrBind); + stats.m_mapped_as = addr.GetMappedAS(m_asmap); if (m_tx_relay != nullptr) { LOCK(m_tx_relay->cs_filter); stats.fRelayTxes = m_tx_relay->fRelayTxes; @@ -1609,7 +1611,7 @@ void CConnman::ThreadDNSAddressSeed() continue; } unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed - if (LookupHost(host.c_str(), vIPs, nMaxIPs, true)) { + if (LookupHost(host, vIPs, nMaxIPs, true)) { for (const CNetAddr& ip : vIPs) { int nOneDay = 24*3600; CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); @@ -1768,7 +1770,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // but inbound and addnode peers do not use our outbound slots. Inbound peers // also have the added issue that they're attacker controlled and could be used // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup()); + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); if (pnode->m_tx_relay == nullptr) { nOutboundBlockRelay++; } else if (!pnode->fFeeler) { @@ -1816,7 +1818,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup())) { + if (!fFeeler && setConnected.count(addr.GetGroup(addrman.m_asmap))) { break; } @@ -1907,7 +1909,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() } for (const std::string& strAddNode : lAddresses) { - CService service(LookupNumeric(strAddNode.c_str(), Params().GetDefaultPort())); + CService service(LookupNumeric(strAddNode, Params().GetDefaultPort())); AddedNodeInfo addedNode{strAddNode, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port @@ -2501,7 +2503,7 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) vstats.reserve(vNodes.size()); for (CNode* pnode : vNodes) { vstats.emplace_back(); - pnode->copyStats(vstats.back()); + pnode->copyStats(vstats.back(), addrman.m_asmap); } } @@ -2788,7 +2790,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup()); + std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.m_asmap)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -153,6 +153,7 @@ public: bool m_use_addrman_outgoing = true; std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; + std::vector<bool> m_asmap; }; void Init(const Options& connOptions) { @@ -330,6 +331,8 @@ public: */ int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } + private: struct ListenSocket { public: @@ -384,10 +387,10 @@ private: static bool NodeFullyConnected(const CNode* pnode); // Network usage totals - CCriticalSection cs_totalBytesRecv; - CCriticalSection cs_totalBytesSent; - uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv); - uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent); + RecursiveMutex cs_totalBytesRecv; + RecursiveMutex cs_totalBytesSent; + uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv) {0}; + uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; // outbound limit & stats uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent); @@ -410,12 +413,12 @@ private: bool fAddressesInitialized{false}; CAddrMan addrman; std::deque<std::string> vOneShots GUARDED_BY(cs_vOneShots); - CCriticalSection cs_vOneShots; + RecursiveMutex cs_vOneShots; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); - CCriticalSection cs_vAddedNodes; + RecursiveMutex cs_vAddedNodes; std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); std::list<CNode*> vNodesDisconnected; - mutable CCriticalSection cs_vNodes; + mutable RecursiveMutex cs_vNodes; std::atomic<NodeId> nLastNodeId{0}; unsigned int nPrevNodeCount{0}; @@ -565,7 +568,7 @@ struct LocalServiceInfo { int nPort; }; -extern CCriticalSection cs_mapLocalHost; +extern RecursiveMutex cs_mapLocalHost; extern std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); extern const std::string NET_MESSAGE_COMMAND_OTHER; @@ -603,6 +606,7 @@ public: CAddress addr; // Bind address of our side of the connection CAddress addrBind; + uint32_t m_mapped_as; }; @@ -713,15 +717,15 @@ public: size_t nSendOffset{0}; // offset inside the first vSendMsg already sent uint64_t nSendBytes GUARDED_BY(cs_vSend){0}; std::deque<std::vector<unsigned char>> vSendMsg GUARDED_BY(cs_vSend); - CCriticalSection cs_vSend; - CCriticalSection cs_hSocket; - CCriticalSection cs_vRecv; + RecursiveMutex cs_vSend; + RecursiveMutex cs_hSocket; + RecursiveMutex cs_vRecv; - CCriticalSection cs_vProcessMsg; + RecursiveMutex cs_vProcessMsg; std::list<CNetMessage> vProcessMsg GUARDED_BY(cs_vProcessMsg); size_t nProcessQueueSize{0}; - CCriticalSection cs_sendProcessing; + RecursiveMutex cs_sendProcessing; std::deque<CInv> vRecvGetData; uint64_t nRecvBytes GUARDED_BY(cs_vRecv){0}; @@ -787,11 +791,11 @@ public: // There is no final sorting before sending, as they are always sent immediately // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); - CCriticalSection cs_inventory; + RecursiveMutex cs_inventory; struct TxRelay { TxRelay() { pfilter = MakeUnique<CBloomFilter>(); } - mutable CCriticalSection cs_filter; + mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs @@ -799,7 +803,7 @@ public: bool fRelayTxes GUARDED_BY(cs_filter){false}; std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter); - mutable CCriticalSection cs_tx_inventory; + mutable RecursiveMutex cs_tx_inventory; CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; // Set of transaction ids we still have to announce. // They are sorted by the mempool before relay, so the order is not important. @@ -810,7 +814,7 @@ public: std::atomic<std::chrono::seconds> m_last_mempool_req{std::chrono::seconds{0}}; std::chrono::microseconds nNextInvSend{0}; - CCriticalSection cs_feeFilter; + RecursiveMutex cs_feeFilter; // Minimum fee rate with which to filter inv's to this node CAmount minFeeFilter GUARDED_BY(cs_feeFilter){0}; CAmount lastSentFeeFilter{0}; @@ -872,12 +876,12 @@ private: NetPermissionFlags m_permissionFlags{ PF_NONE }; std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread - mutable CCriticalSection cs_addrName; + mutable RecursiveMutex cs_addrName; std::string addrName GUARDED_BY(cs_addrName); // Our address, as reported by the peer CService addrLocal GUARDED_BY(cs_addrLocal); - mutable CCriticalSection cs_addrLocal; + mutable RecursiveMutex cs_addrLocal; public: NodeId GetId() const { @@ -979,7 +983,7 @@ public: void CloseSocketDisconnect(); - void copyStats(CNodeStats &stats); + void copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap); ServiceFlags GetLocalServices() const { diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index ef6c40ce20..22fa5ee73b 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -71,7 +71,7 @@ bool NetWhitebindPermissions::TryParse(const std::string str, NetWhitebindPermis const std::string strBind = str.substr(offset); CService addrBind; - if (!Lookup(strBind.c_str(), addrBind, 0, false)) { + if (!Lookup(strBind, addrBind, 0, false)) { error = ResolveErrMsg("whitebind", strBind); return false; } @@ -94,7 +94,7 @@ bool NetWhitelistPermissions::TryParse(const std::string str, NetWhitelistPermis const std::string net = str.substr(offset); CSubNet subnet; - LookupSubNet(net.c_str(), subnet); + LookupSubNet(net, subnet); if (!subnet.IsValid()) { error = strprintf(_("Invalid netmask specified in -whitelist: '%s'").translated, net); return false; diff --git a/src/net_permissions.h b/src/net_permissions.h index b3987de65f..a06d2f544d 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net_processing.cpp b/src/net_processing.cpp index f42a26ca3e..cf4aee0647 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -29,6 +29,7 @@ #include <util/validation.h> #include <memory> +#include <typeinfo> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." @@ -89,7 +90,7 @@ struct COrphanTx { int64_t nTimeExpire; size_t list_pos; }; -CCriticalSection g_cs_orphans; +RecursiveMutex g_cs_orphans; std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); void EraseOrphansFor(NodeId peer); @@ -147,6 +148,14 @@ namespace { std::unique_ptr<CRollingBloomFilter> recentRejects GUARDED_BY(cs_main); uint256 hashRecentRejectsChainTip GUARDED_BY(cs_main); + /* + * Filter for transactions that have been recently confirmed. + * We use this to avoid requesting transactions that have already been + * confirnmed. + */ + RecursiveMutex g_cs_recent_confirmed_transactions; + std::unique_ptr<CRollingBloomFilter> g_recent_confirmed_transactions GUARDED_BY(g_cs_recent_confirmed_transactions); + /** Blocks that are in flight, and that are in the queue to be downloaded. */ struct QueuedBlock { uint256 hash; @@ -989,7 +998,7 @@ static bool TxRelayMayResultInDisconnect(const TxValidationState& state) { /** * Potentially ban a node based on the contents of a BlockValidationState object * - * @param[in] via_compact_block: this bool is passed in because net_processing should + * @param[in] via_compact_block this bool is passed in because net_processing should * punish peers differently depending on whether the data was provided in a compact * block message or not. If the compact block had a valid header, but contained invalid * txs, the peer should not be punished. See BIP 152. @@ -1115,6 +1124,16 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + // Blocks don't typically have more than 4000 transactions, so this should + // be at least six blocks (~1 hr) worth of transactions that we can store. + // If the number of transactions appearing in a block goes up, or if we are + // seeing getdata requests more than an hour after initial announcement, we + // can increase this number. + // The false positive rate of 1/1M should come out to less than 1 + // transaction per day that would be inadvertently ignored (which is the + // same probability that we have in the reject filter). + g_recent_confirmed_transactions.reset(new CRollingBloomFilter(24000, 0.000001)); + const Consensus::Params& consensusParams = Params().GetConsensus(); // Stale tip checking and peer eviction are on two different timers, but we // don't want them to get out of sync due to drift in the scheduler, so we @@ -1128,40 +1147,63 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS * Evict orphan txn pool entries (EraseOrphanTx) based on a newly connected * block. Also save the time of the last tip update. */ -void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { - LOCK(g_cs_orphans); +void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) +{ + { + LOCK(g_cs_orphans); - std::vector<uint256> vOrphanErase; + std::vector<uint256> vOrphanErase; - for (const CTransactionRef& ptx : pblock->vtx) { - const CTransaction& tx = *ptx; + for (const CTransactionRef& ptx : pblock->vtx) { + const CTransaction& tx = *ptx; - // Which orphan pool entries must we evict? - for (const auto& txin : tx.vin) { - auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); - if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const CTransaction& orphanTx = *(*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); - vOrphanErase.push_back(orphanHash); + // Which orphan pool entries must we evict? + for (const auto& txin : tx.vin) { + auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = *(*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } } } - } - // Erase orphan transactions included or precluded by this block - if (vOrphanErase.size()) { - int nErased = 0; - for (const uint256& orphanHash : vOrphanErase) { - nErased += EraseOrphanTx(orphanHash); + // Erase orphan transactions included or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + for (const uint256& orphanHash : vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } - LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); + + g_last_tip_update = GetTime(); } + { + LOCK(g_cs_recent_confirmed_transactions); + for (const auto& ptx : pblock->vtx) { + g_recent_confirmed_transactions->insert(ptx->GetHash()); + } + } +} - g_last_tip_update = GetTime(); +void PeerLogicValidation::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) +{ + // To avoid relay problems with transactions that were previously + // confirmed, clear our filter of recently confirmed transactions whenever + // there's a reorg. + // This means that in a 1-block reorg (where 1 block is disconnected and + // then another block reconnected), our filter will drop to having only one + // block's worth of transactions in it, but that should be fine, since + // presumably the most common case of relaying a confirmed transaction + // should be just after a new block containing it is found. + LOCK(g_cs_recent_confirmed_transactions); + g_recent_confirmed_transactions->reset(); } // All of the following cache a recent block, and are protected by cs_most_recent_block -static CCriticalSection cs_most_recent_block; +static RecursiveMutex cs_most_recent_block; static std::shared_ptr<const CBlock> most_recent_block GUARDED_BY(cs_most_recent_block); static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block GUARDED_BY(cs_most_recent_block); static uint256 most_recent_block_hash GUARDED_BY(cs_most_recent_block); @@ -1310,12 +1352,14 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) LOCK(g_cs_orphans); if (mapOrphanTransactions.count(inv.hash)) return true; } - const CCoinsViewCache& coins_cache = ::ChainstateActive().CoinsTip(); + + { + LOCK(g_cs_recent_confirmed_transactions); + if (g_recent_confirmed_transactions->contains(inv.hash)) return true; + } return recentRejects->contains(inv.hash) || - mempool.exists(inv.hash) || - coins_cache.HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 - coins_cache.HaveCoinInCache(COutPoint(inv.hash, 1)); + mempool.exists(inv.hash); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -3333,32 +3377,10 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return false; if (!pfrom->vRecvGetData.empty()) fMoreWork = true; - } - catch (const std::ios_base::failure& e) - { - if (strstr(e.what(), "end of data")) { - // Allow exceptions from under-length message on vRecv - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "size too large")) { - // Allow exceptions from over-long size - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { - // Allow exceptions from non-canonical encoding - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "Superfluous witness record")) { - // Allow exceptions from illegal witness encoding - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else if (strstr(e.what(), "Unknown transaction optional data")) { - // Allow exceptions from unknown witness encoding - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } else { - PrintExceptionContinue(&e, "ProcessMessages()"); - } - } - catch (const std::exception& e) { - PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (const std::exception& e) { + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what(), typeid(e).name()); } catch (...) { - PrintExceptionContinue(nullptr, "ProcessMessages()"); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(strCommand), nMessageSize); } if (!fRet) { diff --git a/src/net_processing.h b/src/net_processing.h index 4adb7d3a21..6f26abc209 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,7 @@ #include <consensus/params.h> #include <sync.h> -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; @@ -33,6 +33,7 @@ public: * Overridden from CValidationInterface. */ void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; + void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) override; /** * Overridden from CValidationInterface. */ diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 4fbfa2b5c8..1cac57a817 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netaddress.h> #include <hash.h> #include <util/strencodings.h> +#include <util/asmap.h> #include <tinyformat.h> static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; @@ -400,6 +401,73 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } +bool CNetAddr::HasLinkedIPv4() const +{ + return IsRoutable() && (IsIPv4() || IsRFC6145() || IsRFC6052() || IsRFC3964() || IsRFC4380()); +} + +uint32_t CNetAddr::GetLinkedIPv4() const +{ + if (IsIPv4() || IsRFC6145() || IsRFC6052()) { + // IPv4, mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address + return ReadBE32(ip + 12); + } else if (IsRFC3964()) { + // 6to4 tunneled IPv4: the IPv4 address is in bytes 2-6 + return ReadBE32(ip + 2); + } else if (IsRFC4380()) { + // Teredo tunneled IPv4: the IPv4 address is in the last 4 bytes of the address, but bitflipped + return ~ReadBE32(ip + 12); + } + assert(false); +} + +uint32_t CNetAddr::GetNetClass() const { + uint32_t net_class = NET_IPV6; + if (IsLocal()) { + net_class = 255; + } + if (IsInternal()) { + net_class = NET_INTERNAL; + } else if (!IsRoutable()) { + net_class = NET_UNROUTABLE; + } else if (HasLinkedIPv4()) { + net_class = NET_IPV4; + } else if (IsTor()) { + net_class = NET_ONION; + } + return net_class; +} + +uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { + uint32_t net_class = GetNetClass(); + if (asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { + return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. + } + std::vector<bool> ip_bits(128); + if (HasLinkedIPv4()) { + // For lookup, treat as if it was just an IPv4 address (pchIPv4 prefix + IPv4 bits) + for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (pchIPv4[byte_i] >> (7 - bit_i)) & 1; + } + } + uint32_t ipv4 = GetLinkedIPv4(); + for (int i = 0; i < 32; ++i) { + ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1; + } + } else { + // Use all 128 bits of the IPv6 address otherwise + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + } + uint32_t mapped_as = Interpret(asmap, ip_bits); + return mapped_as; +} + /** * Get the canonical identifier of our network group * @@ -410,67 +478,51 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const * @note No two connections will be attempted to addresses with the same network * group. */ -std::vector<unsigned char> CNetAddr::GetGroup() const +std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) const { std::vector<unsigned char> vchRet; - int nClass = NET_IPV6; + uint32_t net_class = GetNetClass(); + // If non-empty asmap is supplied and the address is IPv4/IPv6, + // return ASN to be used for bucketing. + uint32_t asn = GetMappedAS(asmap); + if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). + vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + + vchRet.push_back(net_class); int nStartByte = 0; int nBits = 16; - // all local addresses belong to the same group - if (IsLocal()) - { - nClass = 255; + if (IsLocal()) { + // all local addresses belong to the same group nBits = 0; - } - // all internal-usage addresses get their own group - if (IsInternal()) - { - nClass = NET_INTERNAL; + } else if (IsInternal()) { + // all internal-usage addresses get their own group nStartByte = sizeof(g_internal_prefix); nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; - } - // all other unroutable addresses belong to the same group - else if (!IsRoutable()) - { - nClass = NET_UNROUTABLE; + } else if (!IsRoutable()) { + // all other unroutable addresses belong to the same group nBits = 0; - } - // for IPv4 addresses, '1' + the 16 higher-order bits of the IP - // includes mapped IPv4, SIIT translated IPv4, and the well-known prefix - else if (IsIPv4() || IsRFC6145() || IsRFC6052()) - { - nClass = NET_IPV4; - nStartByte = 12; - } - // for 6to4 tunnelled addresses, use the encapsulated IPv4 address - else if (IsRFC3964()) - { - nClass = NET_IPV4; - nStartByte = 2; - } - // for Teredo-tunnelled IPv6 addresses, use the encapsulated IPv4 address - else if (IsRFC4380()) - { - vchRet.push_back(NET_IPV4); - vchRet.push_back(GetByte(3) ^ 0xFF); - vchRet.push_back(GetByte(2) ^ 0xFF); + } else if (HasLinkedIPv4()) { + // IPv4 addresses (and mapped IPv4 addresses) use /16 groups + uint32_t ipv4 = GetLinkedIPv4(); + vchRet.push_back((ipv4 >> 24) & 0xFF); + vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; - } - else if (IsTor()) - { - nClass = NET_ONION; + } else if (IsTor()) { nStartByte = 6; nBits = 4; - } - // for he.net, use /36 groups - else if (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70) + } else if (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70) { + // for he.net, use /36 groups nBits = 36; - // for the rest of the IPv6 network, use /32 groups - else + } else { + // for the rest of the IPv6 network, use /32 groups nBits = 32; - - vchRet.push_back(nClass); + } // push our ip onto vchRet byte by byte... while (nBits >= 8) diff --git a/src/netaddress.h b/src/netaddress.h index 32f4476fac..b300b709f3 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,7 +39,6 @@ class CNetAddr explicit CNetAddr(const struct in_addr& ipv4Addr); void SetIP(const CNetAddr& ip); - private: /** * Set raw IPv4 or IPv6 address (in network byte order) * @note Only NET_IPV4 and NET_IPV6 are allowed for network. @@ -78,7 +77,19 @@ class CNetAddr unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; - std::vector<unsigned char> GetGroup() const; + uint32_t GetNetClass() const; + + //! For IPv4, mapped IPv4, SIIT translated IPv4, Teredo, 6to4 tunneled addresses, return the relevant IPv4 address as a uint32. + uint32_t GetLinkedIPv4() const; + //! Whether this address has a linked IPv4 address (see GetLinkedIPv4()). + bool HasLinkedIPv4() const; + + // The AS on the BGP path to the node we use to diversify + // peers in AddrMan bucketing based on the AS infrastructure. + // The ip->AS mapping depends on how asmap is constructed. + uint32_t GetMappedAS(const std::vector<bool> &asmap) const; + + std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); diff --git a/src/netbase.cpp b/src/netbase.cpp index d1cde8c40f..a70179cb16 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,9 @@ #include <sync.h> #include <tinyformat.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/string.h> +#include <util/system.h> #include <atomic> @@ -27,7 +28,7 @@ #endif // Settings -static CCriticalSection cs_proxyInfos; +static RecursiveMutex cs_proxyInfos; static proxyType proxyInfo[NET_MAX] GUARDED_BY(cs_proxyInfos); static proxyType nameProxy GUARDED_BY(cs_proxyInfos); int nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -59,10 +60,14 @@ std::string GetNetworkName(enum Network net) { } } -bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) +bool static LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { vIP.clear(); + if (!ValidAsCString(name)) { + return false; + } + { CNetAddr addr; // From our perspective, onion addresses are not hostnames but rather @@ -71,7 +76,7 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign // getaddrinfo to decode them and it wouldn't make sense to resolve // them, we return a network address representing it instead. See // CNetAddr::SetSpecial(const std::string&) for more details. - if (addr.SetSpecial(std::string(pszName))) { + if (addr.SetSpecial(name)) { vIP.push_back(addr); return true; } @@ -93,7 +98,7 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign // hostname lookups. aiHint.ai_flags = fAllowLookup ? AI_ADDRCONFIG : AI_NUMERICHOST; struct addrinfo *aiRes = nullptr; - int nErr = getaddrinfo(pszName, nullptr, &aiHint, &aiRes); + int nErr = getaddrinfo(name.c_str(), nullptr, &aiHint, &aiRes); if (nErr) return false; @@ -131,7 +136,7 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign /** * Resolve a host string to its corresponding network addresses. * - * @param pszName The string representing a host. Could be a name or a numerical + * @param name The string representing a host. Could be a name or a numerical * IP address (IPv6 addresses in their bracketed form are * allowed). * @param[out] vIP The resulting network addresses to which the specified host @@ -143,28 +148,34 @@ bool static LookupIntern(const char *pszName, std::vector<CNetAddr>& vIP, unsign * @see Lookup(const char *, std::vector<CService>&, int, bool, unsigned int) * for additional parameter descriptions. */ -bool LookupHost(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) +bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) { - std::string strHost(pszName); + if (!ValidAsCString(name)) { + return false; + } + std::string strHost = name; if (strHost.empty()) return false; if (strHost.front() == '[' && strHost.back() == ']') { strHost = strHost.substr(1, strHost.size() - 2); } - return LookupIntern(strHost.c_str(), vIP, nMaxSolutions, fAllowLookup); + return LookupIntern(strHost, vIP, nMaxSolutions, fAllowLookup); } /** * Resolve a host string to its first corresponding network address. * - * @see LookupHost(const char *, std::vector<CNetAddr>&, unsigned int, bool) for + * @see LookupHost(const std::string&, std::vector<CNetAddr>&, unsigned int, bool) for * additional parameter descriptions. */ -bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup) +bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup) { + if (!ValidAsCString(name)) { + return false; + } std::vector<CNetAddr> vIP; - LookupHost(pszName, vIP, 1, fAllowLookup); + LookupHost(name, vIP, 1, fAllowLookup); if(vIP.empty()) return false; addr = vIP.front(); @@ -174,7 +185,7 @@ bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup) /** * Resolve a service string to its corresponding service. * - * @param pszName The string representing a service. Could be a name or a + * @param name The string representing a service. Could be a name or a * numerical IP address (IPv6 addresses should be in their * disambiguated bracketed form), optionally followed by a port * number. (e.g. example.com:8333 or @@ -191,16 +202,17 @@ bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup) * @returns Whether or not the service string successfully resolved to any * resulting services. */ -bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions) +bool Lookup(const std::string& name, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions) { - if (pszName[0] == 0) + if (name.empty() || !ValidAsCString(name)) { return false; + } int port = portDefault; std::string hostname; - SplitHostPort(std::string(pszName), port, hostname); + SplitHostPort(name, port, hostname); std::vector<CNetAddr> vIP; - bool fRet = LookupIntern(hostname.c_str(), vIP, nMaxSolutions, fAllowLookup); + bool fRet = LookupIntern(hostname, vIP, nMaxSolutions, fAllowLookup); if (!fRet) return false; vAddr.resize(vIP.size()); @@ -215,10 +227,13 @@ bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, * @see Lookup(const char *, std::vector<CService>&, int, bool, unsigned int) * for additional parameter descriptions. */ -bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLookup) +bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllowLookup) { + if (!ValidAsCString(name)) { + return false; + } std::vector<CService> vService; - bool fRet = Lookup(pszName, vService, portDefault, fAllowLookup, 1); + bool fRet = Lookup(name, vService, portDefault, fAllowLookup, 1); if (!fRet) return false; addr = vService[0]; @@ -235,12 +250,15 @@ bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLoo * @see Lookup(const char *, CService&, int, bool) for additional parameter * descriptions. */ -CService LookupNumeric(const char *pszName, int portDefault) +CService LookupNumeric(const std::string& name, int portDefault) { + if (!ValidAsCString(name)) { + return {}; + } CService addr; // "1.2:345" will fail to resolve the ip, but will still set the port. // If the ip fails to resolve, re-init the result. - if(!Lookup(pszName, addr, portDefault, false)) + if(!Lookup(name, addr, portDefault, false)) addr = CService(); return addr; } @@ -763,17 +781,16 @@ bool IsProxy(const CNetAddr &addr) { * @param hSocket The socket on which to connect to the SOCKS5 proxy. * @param nTimeout Wait this many milliseconds for the connection to the SOCKS5 * proxy to be established. - * @param outProxyConnectionFailed[out] Whether or not the connection to the + * @param[out] outProxyConnectionFailed Whether or not the connection to the * SOCKS5 proxy failed. * * @returns Whether or not the operation succeeded. */ -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool *outProxyConnectionFailed) +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool& outProxyConnectionFailed) { // first connect to proxy server if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout, true)) { - if (outProxyConnectionFailed) - *outProxyConnectionFailed = true; + outProxyConnectionFailed = true; return false; } // do socks negotiation @@ -796,23 +813,25 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int * Parse and resolve a specified subnet string into the appropriate internal * representation. * - * @param pszName A string representation of a subnet of the form `network + * @param strSubnet A string representation of a subnet of the form `network * address [ "/", ( CIDR-style suffix | netmask ) ]`(e.g. * `2001:db8::/32`, `192.0.2.0/255.255.255.0`, or `8.8.8.8`). * @param ret The resulting internal representation of a subnet. * * @returns Whether the operation succeeded or not. */ -bool LookupSubNet(const char* pszName, CSubNet& ret) +bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) { - std::string strSubnet(pszName); + if (!ValidAsCString(strSubnet)) { + return false; + } size_t slash = strSubnet.find_last_of('/'); std::vector<CNetAddr> vIP; std::string strAddress = strSubnet.substr(0, slash); - // TODO: Use LookupHost(const char *, CNetAddr&, bool) instead to just get + // TODO: Use LookupHost(const std::string&, CNetAddr&, bool) instead to just get // one CNetAddr. - if (LookupHost(strAddress.c_str(), vIP, 1, false)) + if (LookupHost(strAddress, vIP, 1, false)) { CNetAddr network = vIP[0]; if (slash != strSubnet.npos) @@ -827,7 +846,7 @@ bool LookupSubNet(const char* pszName, CSubNet& ret) else // If not a valid number, try full netmask syntax { // Never allow lookup for netmask - if (LookupHost(strNetmask.c_str(), vIP, 1, false)) { + if (LookupHost(strNetmask, vIP, 1, false)) { ret = CSubNet(network, vIP[0]); return ret.IsValid(); } diff --git a/src/netbase.h b/src/netbase.h index 313a575687..ac4cd97673 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,15 +45,15 @@ bool IsProxy(const CNetAddr &addr); bool SetNameProxy(const proxyType &addrProxy); bool HaveNameProxy(); bool GetNameProxy(proxyType &nameProxyOut); -bool LookupHost(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup); -bool LookupHost(const char *pszName, CNetAddr& addr, bool fAllowLookup); -bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLookup); -bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); -CService LookupNumeric(const char *pszName, int portDefault = 0); -bool LookupSubNet(const char *pszName, CSubNet& subnet); +bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup); +bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup); +bool Lookup(const std::string& name, CService& addr, int portDefault, bool fAllowLookup); +bool Lookup(const std::string& name, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); +CService LookupNumeric(const std::string& name, int portDefault = 0); +bool LookupSubNet(const std::string& strSubnet, CSubNet& subnet); SOCKET CreateSocket(const CService &addrConnect); bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocketRet, int nTimeout, bool manual_connection); -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed); +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool& outProxyConnectionFailed); /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); /** Close socket and set hSocket to INVALID_SOCKET */ diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 9a30c3f083..8678b33cf3 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -1,7 +1,8 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <amount.h> #include <coins.h> #include <consensus/tx_verify.h> #include <node/psbt.h> @@ -31,9 +32,17 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) // Check for a UTXO CTxOut utxo; if (psbtx.GetInputUTXO(utxo, i)) { + if (!MoneyRange(utxo.nValue) || !MoneyRange(in_amt + utxo.nValue)) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u has invalid value", i)); + return result; + } in_amt += utxo.nValue; input_analysis.has_utxo = true; } else { + if (input.non_witness_utxo && psbtx.tx->vin[i].prevout.n >= input.non_witness_utxo->vout.size()) { + result.SetInvalid(strprintf("PSBT is not valid. Input %u specifies invalid prevout", i)); + return result; + } input_analysis.has_utxo = false; input_analysis.is_final = false; input_analysis.next = PSBTRole::UPDATER; @@ -85,9 +94,16 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) // Get the output amount CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), CAmount(0), [](CAmount a, const CTxOut& b) { + if (!MoneyRange(a) || !MoneyRange(b.nValue) || !MoneyRange(a + b.nValue)) { + return CAmount(-1); + } return a += b.nValue; } ); + if (!MoneyRange(out_amt)) { + result.SetInvalid(strprintf("PSBT is not valid. Output amount invalid")); + return result; + } // Get the fee CAmount fee = in_amt - out_amt; diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 5e2e502015..1bb9b88d00 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/transaction.h b/src/node/transaction.h index 35873d8376..a85dfb8ace 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -22,10 +22,10 @@ struct NodeContext; * * @param[in] node reference to node context * @param[in] tx the transaction to broadcast - * @param[out] &err_string reference to std::string to fill with error string if available + * @param[out] err_string reference to std::string to fill with error string if available * @param[in] max_tx_fee reject txs with fees higher than this (if 0, accept any fee) * @param[in] relay flag if both mempool insertion and p2p relay are requested - * @param[in] wait_callback, wait until callbacks have been processed to avoid stale result due to a sequentially RPC. + * @param[in] wait_callback wait until callbacks have been processed to avoid stale result due to a sequentially RPC. * return error */ NODISCARD TransactionError BroadcastTransaction(NodeContext& node, CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); diff --git a/src/noui.cpp b/src/noui.cpp index a5b7a2d591..11cfe7f94d 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/noui.h b/src/noui.h index 621e9c2798..5e5767b453 100644 --- a/src/noui.h +++ b/src/noui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2018 The Bitcoin Core developers +// Copyright (c) 2013-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/optional.h b/src/optional.h index 95a3b24d0a..a382cd7b77 100644 --- a/src/optional.h +++ b/src/optional.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 5cc43898a7..567eecb5c9 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +19,8 @@ static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; +const std::array<OutputType, 3> OUTPUT_TYPES = {OutputType::LEGACY, OutputType::P2SH_SEGWIT, OutputType::BECH32}; + bool ParseOutputType(const std::string& type, OutputType& output_type) { if (type == OUTPUT_TYPE_STRING_LEGACY) { @@ -80,22 +82,30 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, { // Add script to keystore keystore.AddCScript(script); + ScriptHash sh(script); // Note that scripts over 520 bytes are not yet supported. switch (type) { case OutputType::LEGACY: - return ScriptHash(script); + keystore.AddCScript(GetScriptForDestination(sh)); + return sh; case OutputType::P2SH_SEGWIT: case OutputType::BECH32: { CTxDestination witdest = WitnessV0ScriptHash(script); CScript witprog = GetScriptForDestination(witdest); // Check if the resulting program is solvable (i.e. doesn't use an uncompressed key) - if (!IsSolvable(keystore, witprog)) return ScriptHash(script); + if (!IsSolvable(keystore, witprog)) { + // Since the wsh is invalid, add and return the sh instead. + keystore.AddCScript(GetScriptForDestination(sh)); + return sh; + } // Add the redeemscript, so that P2WSH and P2SH-P2WSH outputs are recognized as ours. keystore.AddCScript(witprog); if (type == OutputType::BECH32) { return witdest; } else { - return ScriptHash(witprog); + ScriptHash sh_w = ScriptHash(witprog); + keystore.AddCScript(GetScriptForDestination(sh_w)); + return sh_w; } } default: assert(false); diff --git a/src/outputtype.h b/src/outputtype.h index 6acbaa2f3e..1438f65844 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,6 +10,7 @@ #include <script/signingprovider.h> #include <script/standard.h> +#include <array> #include <string> #include <vector> @@ -27,6 +28,8 @@ enum class OutputType { CHANGE_AUTO, }; +extern const std::array<OutputType, 3> OUTPUT_TYPES; + NODISCARD bool ParseOutputType(const std::string& str, OutputType& output_type); const std::string& FormatOutputType(OutputType type); @@ -47,4 +50,3 @@ std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, const CScript& script, OutputType); #endif // BITCOIN_OUTPUTTYPE_H - diff --git a/src/policy/feerate.h b/src/policy/feerate.h index d081f2ce8e..c040867965 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index a66e4464db..25458eead2 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/fees.h b/src/policy/fees.h index 16683bf5ad..6ee6e0d547 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICY_FEES_H @@ -223,7 +223,7 @@ public: unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const; private: - mutable CCriticalSection m_cs_fee_estimator; + mutable RecursiveMutex m_cs_fee_estimator; unsigned int nBestSeenHeight GUARDED_BY(m_cs_fee_estimator); unsigned int firstRecordedHeight GUARDED_BY(m_cs_fee_estimator); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 51de5841ec..07d51c0088 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/policy.h b/src/policy/policy.h index ebe040f0ea..1561a41c5e 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index b4b8341d77..f8b17d18d5 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 0707b0044f..d335fbbb36 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/settings.cpp b/src/policy/settings.cpp index e8e1559407..eb2ec56850 100644 --- a/src/policy/settings.cpp +++ b/src/policy/settings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/policy/settings.h b/src/policy/settings.h index 30a7189c93..0b4fc1e770 100644 --- a/src/policy/settings.h +++ b/src/policy/settings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/prevector.h b/src/prevector.h index 4fb07688ff..f4ece738a8 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 0c84ed6da2..50a30cb511 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index aad991e2f1..00ccbc32f9 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.cpp b/src/psbt.cpp index 9ede62efdf..e6b6285652 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -66,8 +66,11 @@ bool PartiallySignedTransaction::AddOutput(const CTxOut& txout, const PSBTOutput bool PartiallySignedTransaction::GetInputUTXO(CTxOut& utxo, int input_index) const { PSBTInput input = inputs[input_index]; - int prevout_index = tx->vin[input_index].prevout.n; + uint32_t prevout_index = tx->vin[input_index].prevout.n; if (input.non_witness_utxo) { + if (prevout_index >= input.non_witness_utxo->vout.size()) { + return false; + } utxo = input.non_witness_utxo->vout[prevout_index]; } else if (!input.witness_utxo.IsNull()) { utxo = input.witness_utxo; @@ -255,6 +258,9 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& if (input.non_witness_utxo) { // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. COutPoint prevout = tx.vin[index].prevout; + if (prevout.n >= input.non_witness_utxo->vout.size()) { + return false; + } if (input.non_witness_utxo->GetHash() != prevout.hash) { return false; } diff --git a/src/psbt.h b/src/psbt.h index d507d5b6b7..dfba261961 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -584,7 +584,7 @@ void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransactio /** * Finalizes a PSBT if possible, combining partial signatures. * - * @param[in,out] &psbtx reference to PartiallySignedTransaction to finalize + * @param[in,out] psbtx PartiallySignedTransaction to finalize * return True if the PSBT is now complete, false otherwise */ bool FinalizePSBT(PartiallySignedTransaction& psbtx); @@ -592,7 +592,7 @@ bool FinalizePSBT(PartiallySignedTransaction& psbtx); /** * Finalizes a PSBT if possible, and extracts it to a CMutableTransaction if it could be finalized. * - * @param[in] &psbtx reference to PartiallySignedTransaction + * @param[in] psbtx PartiallySignedTransaction * @param[out] result CMutableTransaction representing the complete transaction, if successful * @return True if we successfully extracted the transaction, false otherwise */ @@ -601,7 +601,7 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti /** * Combines PSBTs with the same underlying transaction, resulting in a single PSBT with all partial signatures from each input. * - * @param[out] &out the combined PSBT, if successful + * @param[out] out the combined PSBT, if successful * @param[in] psbtxs the PSBTs to combine * @return error (OK if we successfully combined the transactions, other error if they were not compatible) */ diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 21e51a380d..ef42aa5bc7 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/pubkey.h b/src/pubkey.h index 76f743da66..2fc92c9bc6 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index d8c39e8862..1aaf33c6a4 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 131cceccbe..3ac98a5970 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index 2ababb5e1e..67e7704551 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index bdfd3fb9a0..20fc5045ae 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index b6c6984b10..72f16bb09f 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,12 +6,13 @@ #include <interfaces/node.h> #include <net_types.h> // For banmap_t -#include <qt/clientmodel.h> #include <utility> -#include <QDebug> +#include <QDateTime> #include <QList> +#include <QModelIndex> +#include <QVariant> bool BannedNodeLessThan::operator()(const CCombinedBan& left, const CCombinedBan& right) const { @@ -78,10 +79,9 @@ public: } }; -BanTableModel::BanTableModel(interfaces::Node& node, ClientModel *parent) : +BanTableModel::BanTableModel(interfaces::Node& node, QObject* parent) : QAbstractTableModel(parent), - m_node(node), - clientModel(parent) + m_node(node) { columns << tr("IP/Netmask") << tr("Banned Until"); priv.reset(new BanTablePriv()); diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 9dec5fa6a9..f01c506a1e 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -12,7 +12,6 @@ #include <QAbstractTableModel> #include <QStringList> -class ClientModel; class BanTablePriv; namespace interfaces { @@ -45,7 +44,7 @@ class BanTableModel : public QAbstractTableModel Q_OBJECT public: - explicit BanTableModel(interfaces::Node& node, ClientModel *parent = nullptr); + explicit BanTableModel(interfaces::Node& node, QObject* parent); ~BanTableModel(); void startAutoRefresh(); void stopAutoRefresh(); @@ -72,7 +71,6 @@ public Q_SLOTS: private: interfaces::Node& m_node; - ClientModel *clientModel; QStringList columns; std::unique_ptr<BanTablePriv> priv; }; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 0021c3dbc7..4313d6ee7f 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -281,8 +281,11 @@ void BitcoinApplication::parameterSetup() m_node.initParameterInteraction(); } -void BitcoinApplication::SetPrune(bool prune, bool force) { - optionsModel->SetPrune(prune, force); +void BitcoinApplication::InitializePruneSetting(bool prune) +{ + // If prune is set, intentionally override existing prune size with + // the default size since this is called when choosing a new datadir. + optionsModel->SetPruneTargetGB(prune ? DEFAULT_PRUNE_TARGET_GB : 0, true); } void BitcoinApplication::requestInitialize() @@ -556,12 +559,13 @@ int GuiMain(int argc, char* argv[]) qInstallMessageHandler(DebugMessageHandler); // Allow parameter interaction before we create the options model app.parameterSetup(); + GUIUtil::LogQtInfo(); // Load GUI settings from QSettings app.createOptionsModel(gArgs.GetBoolArg("-resetguisettings", false)); if (did_show_intro) { // Store intro dialog settings other than datadir (network specific) - app.SetPrune(prune, true); + app.InitializePruneSetting(prune); } if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 8c77fd8a7d..077a37fde5 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -67,8 +67,8 @@ public: void parameterSetup(); /// Create options model void createOptionsModel(bool resetSettings); - /// Update prune value - void SetPrune(bool prune, bool force = false); + /// Initialize prune setting + void InitializePruneSetting(bool prune); /// Create main window void createWindow(const NetworkStyle *networkStyle); /// Create splash screen diff --git a/src/qt/bitcoin_locale.qrc b/src/qt/bitcoin_locale.qrc index dec3670536..c781072e9b 100644 --- a/src/qt/bitcoin_locale.qrc +++ b/src/qt/bitcoin_locale.qrc @@ -11,6 +11,7 @@ <file alias="de_DE">locale/bitcoin_de_DE.qm</file> <file alias="el">locale/bitcoin_el.qm</file> <file alias="el_GR">locale/bitcoin_el_GR.qm</file> + <file alias="en">locale/bitcoin_en.qm</file> <file alias="en_AU">locale/bitcoin_en_AU.qm</file> <file alias="en_GB">locale/bitcoin_en_GB.qm</file> <file alias="eo">locale/bitcoin_eo.qm</file> diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 23df1c929a..7acc82370f 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 8444984b27..5fab267610 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -318,8 +318,8 @@ void BitcoinGUI::createActions() verifyMessageAction = new QAction(tr("&Verify message..."), this); verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); - openRPCConsoleAction = new QAction(tr("&Debug window"), this); - openRPCConsoleAction->setStatusTip(tr("Open debugging and diagnostic console")); + openRPCConsoleAction = new QAction(tr("Node window"), this); + openRPCConsoleAction->setStatusTip(tr("Open node debugging and diagnostic console")); // initially disable the debug window menu item openRPCConsoleAction->setEnabled(false); openRPCConsoleAction->setObjectName("openRPCConsoleAction"); @@ -634,10 +634,10 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) void BitcoinGUI::addWallet(WalletModel* walletModel) { if (!walletFrame) return; + if (!walletFrame->addWallet(walletModel)) return; const QString display_name = walletModel->getDisplayName(); setWalletActionsEnabled(true); rpcConsole->addWallet(walletModel); - walletFrame->addWallet(walletModel); m_wallet_selector->addItem(display_name, QVariant::fromValue(walletModel)); if (m_wallet_selector->count() == 2) { m_wallet_selector_label_action->setVisible(true); @@ -648,6 +648,10 @@ void BitcoinGUI::addWallet(WalletModel* walletModel) void BitcoinGUI::removeWallet(WalletModel* walletModel) { if (!walletFrame) return; + + labelWalletHDStatusIcon->hide(); + labelWalletEncryptionIcon->hide(); + int index = m_wallet_selector->findData(QVariant::fromValue(walletModel)); m_wallet_selector->removeItem(index); if (m_wallet_selector->count() == 0) { @@ -1209,7 +1213,7 @@ void BitcoinGUI::setHDStatus(bool privkeyDisabled, int hdEnabled) { labelWalletHDStatusIcon->setPixmap(platformStyle->SingleColorIcon(privkeyDisabled ? ":/icons/eye" : hdEnabled ? ":/icons/hd_enabled" : ":/icons/hd_disabled").pixmap(STATUSBAR_ICONSIZE,STATUSBAR_ICONSIZE)); labelWalletHDStatusIcon->setToolTip(privkeyDisabled ? tr("Private key <b>disabled</b>") : hdEnabled ? tr("HD key generation is <b>enabled</b>") : tr("HD key generation is <b>disabled</b>")); - + labelWalletHDStatusIcon->show(); // eventually disable the QLabel to set its opacity to 50% labelWalletHDStatusIcon->setEnabled(hdEnabled); } diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index b27f8a744f..d9711af123 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 06a1544fa2..4c8a889965 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,22 +13,6 @@ // U+2009 THIN SPACE = UTF-8 E2 80 89 #define REAL_THIN_SP_CP 0x2009 #define REAL_THIN_SP_UTF8 "\xE2\x80\x89" -#define REAL_THIN_SP_HTML " " - -// U+200A HAIR SPACE = UTF-8 E2 80 8A -#define HAIR_SP_CP 0x200A -#define HAIR_SP_UTF8 "\xE2\x80\x8A" -#define HAIR_SP_HTML " " - -// U+2006 SIX-PER-EM SPACE = UTF-8 E2 80 86 -#define SIXPEREM_SP_CP 0x2006 -#define SIXPEREM_SP_UTF8 "\xE2\x80\x86" -#define SIXPEREM_SP_HTML " " - -// U+2007 FIGURE SPACE = UTF-8 E2 80 87 -#define FIGURE_SP_CP 0x2007 -#define FIGURE_SP_UTF8 "\xE2\x80\x87" -#define FIGURE_SP_HTML " " // QMessageBox seems to have a bug whereby it doesn't display thin/hair spaces // correctly. Workaround is to display a space in a small font. If you @@ -114,9 +98,6 @@ public: { text.remove(' '); text.remove(QChar(THIN_SP_CP)); -#if (THIN_SP_CP != REAL_THIN_SP_CP) - text.remove(QChar(REAL_THIN_SP_CP)); -#endif return text; } diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index b84587a99b..e8146982f9 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index d3a95d531e..79175e0af4 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 3302dde4ed..9495ba389a 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 88fc8b704f..39dc9a5e9e 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/csvmodelwriter.h b/src/qt/csvmodelwriter.h index e8611bea35..e443529335 100644 --- a/src/qt/csvmodelwriter.h +++ b/src/qt/csvmodelwriter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index be807b20c0..ebb6bbd4f5 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -11,7 +11,7 @@ </rect> </property> <property name="windowTitle"> - <string>Debug window</string> + <string>Node window</string> </property> <layout class="QVBoxLayout" name="verticalLayout_2"> <item> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index da19a6fa2e..d2e7ca8f06 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -351,6 +351,9 @@ QLabel { color: rgb(40,40,40); }</string> <property name="text"> <string>Hide</string> </property> + <property name="shortcut"> + <string>Esc</string> + </property> <property name="focusPolicy"> <enum>Qt::StrongFocus</enum> </property> diff --git a/src/qt/guiconstants.h b/src/qt/guiconstants.h index dcdb247977..9457ea37d6 100644 --- a/src/qt/guiconstants.h +++ b/src/qt/guiconstants.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -51,4 +51,7 @@ static const int TOOLTIP_WRAP_THRESHOLD = 80; /* One gigabyte (GB) in bytes */ static constexpr uint64_t GB_BYTES{1000000000}; +// Default prune target displayed in GUI. +static constexpr int DEFAULT_PRUNE_TARGET_GB{2}; + #endif // BITCOIN_QT_GUICONSTANTS_H diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 843de651e5..911322092c 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -44,14 +44,20 @@ #include <QFont> #include <QFontDatabase> #include <QFontMetrics> +#include <QGuiApplication> #include <QKeyEvent> #include <QLineEdit> +#include <QList> #include <QMouseEvent> #include <QProgressDialog> +#include <QScreen> #include <QSettings> +#include <QSize> +#include <QString> #include <QTextDocument> // for Qt::mightBeRichText #include <QThread> #include <QUrlQuery> +#include <QtGlobal> #if defined(Q_OS_MAC) @@ -879,4 +885,23 @@ int TextWidth(const QFontMetrics& fm, const QString& text) #endif } +void LogQtInfo() +{ +#ifdef QT_STATIC + const std::string qt_link{"static"}; +#else + const std::string qt_link{"dynamic"}; +#endif +#ifdef QT_STATICPLUGIN + const std::string plugin_link{"static"}; +#else + const std::string plugin_link{"dynamic"}; +#endif + LogPrintf("Qt %s (%s), plugin=%s (%s)\n", qVersion(), qt_link, QGuiApplication::platformName().toStdString(), plugin_link); + LogPrintf("System: %s, %s\n", QSysInfo::prettyProductName().toStdString(), QSysInfo::buildAbi().toStdString()); + for (const QScreen* s : QGuiApplication::screens()) { + LogPrintf("Screen: %s %dx%d, pixel ratio=%.1f\n", s->name().toStdString(), s->size().width(), s->size().height(), s->devicePixelRatio()); + } +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 9db92f94d7..05e73cc5f0 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -265,6 +265,11 @@ namespace GUIUtil * In Qt 5.11 the QFontMetrics::horizontalAdvance() was introduced. */ int TextWidth(const QFontMetrics& fm, const QString& text); + + /** + * Writes to debug.log short info about the used Qt and the host system. + */ + void LogQtInfo(); } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 53c80639b9..ad21dfc3ef 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,7 @@ #include <qt/guiconstants.h> #include <qt/guiutil.h> +#include <qt/optionsmodel.h> #include <interfaces/node.h> #include <util/system.h> @@ -22,9 +23,6 @@ #include <cmath> -/* Total required space (in GB) depending on user choice (prune, not prune) */ -static uint64_t requiredSpace; - /* Check free space asynchronously to prevent hanging the UI thread. Up to one request to check a path is in flight to this thread; when the check() @@ -109,14 +107,24 @@ void FreespaceChecker::check() Q_EMIT reply(replyStatus, replyMessage, freeBytesAvailable); } +namespace { +//! Return pruning size that will be used if automatic pruning is enabled. +int GetPruneTargetGB() +{ + int64_t prune_target_mib = gArgs.GetArg("-prune", 0); + // >1 means automatic pruning is enabled by config, 1 means manual pruning, 0 means no pruning. + return prune_target_mib > 1 ? PruneMiBtoGB(prune_target_mib) : DEFAULT_PRUNE_TARGET_GB; +} +} // namespace -Intro::Intro(QWidget *parent, uint64_t blockchain_size, uint64_t chain_state_size) : +Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : QDialog(parent), ui(new Ui::Intro), thread(nullptr), signalled(false), - m_blockchain_size(blockchain_size), - m_chain_state_size(chain_state_size) + m_blockchain_size_gb(blockchain_size_gb), + m_chain_state_size_gb(chain_state_size_gb), + m_prune_target_gb{GetPruneTargetGB()} { ui->setupUi(this); ui->welcomeLabel->setText(ui->welcomeLabel->text().arg(PACKAGE_NAME)); @@ -124,37 +132,24 @@ Intro::Intro(QWidget *parent, uint64_t blockchain_size, uint64_t chain_state_siz ui->lblExplanation1->setText(ui->lblExplanation1->text() .arg(PACKAGE_NAME) - .arg(m_blockchain_size) + .arg(m_blockchain_size_gb) .arg(2009) .arg(tr("Bitcoin")) ); ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(PACKAGE_NAME)); - uint64_t pruneTarget = std::max<int64_t>(0, gArgs.GetArg("-prune", 0)); - if (pruneTarget > 1) { // -prune=1 means enabled, above that it's a size in MB + if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } - ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(pruneTarget ? pruneTarget / 1000 : 2)); - requiredSpace = m_blockchain_size; - QString storageRequiresMsg = tr("At least %1 GB of data will be stored in this directory, and it will grow over time."); - if (pruneTarget) { - uint64_t prunedGBs = std::ceil(pruneTarget * 1024 * 1024.0 / GB_BYTES); - if (prunedGBs <= requiredSpace) { - requiredSpace = prunedGBs; - storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); - } - ui->lblExplanation3->setVisible(true); - } else { - ui->lblExplanation3->setVisible(false); - } - requiredSpace += m_chain_state_size; - ui->sizeWarningLabel->setText( - tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + - storageRequiresMsg.arg(requiredSpace) + " " + - tr("The wallet will also be stored in this directory.") - ); - this->adjustSize(); + ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(m_prune_target_gb)); + UpdatePruneLabels(ui->prune->isChecked()); + + connect(ui->prune, &QCheckBox::toggled, [this](bool prune_checked) { + UpdatePruneLabels(prune_checked); + UpdateFreeSpaceLabel(); + }); + startThread(); } @@ -270,25 +265,31 @@ void Intro::setStatus(int status, const QString &message, quint64 bytesAvailable { ui->freeSpace->setText(""); } else { - QString freeString = tr("%n GB of free space available", "", bytesAvailable/GB_BYTES); - if(bytesAvailable < requiredSpace * GB_BYTES) - { - freeString += " " + tr("(of %n GB needed)", "", requiredSpace); - ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); - ui->prune->setChecked(true); - } else if (bytesAvailable / GB_BYTES - requiredSpace < 10) { - freeString += " " + tr("(%n GB needed for full chain)", "", requiredSpace); - ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); - ui->prune->setChecked(true); - } else { - ui->freeSpace->setStyleSheet(""); + m_bytes_available = bytesAvailable; + if (ui->prune->isEnabled()) { + ui->prune->setChecked(m_bytes_available < (m_blockchain_size_gb + m_chain_state_size_gb + 10) * GB_BYTES); } - ui->freeSpace->setText(freeString + "."); + UpdateFreeSpaceLabel(); } /* Don't allow confirm in ERROR state */ ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(status != FreespaceChecker::ST_ERROR); } +void Intro::UpdateFreeSpaceLabel() +{ + QString freeString = tr("%n GB of free space available", "", m_bytes_available / GB_BYTES); + if (m_bytes_available < m_required_space_gb * GB_BYTES) { + freeString += " " + tr("(of %n GB needed)", "", m_required_space_gb); + ui->freeSpace->setStyleSheet("QLabel { color: #800000 }"); + } else if (m_bytes_available / GB_BYTES - m_required_space_gb < 10) { + freeString += " " + tr("(%n GB needed for full chain)", "", m_required_space_gb); + ui->freeSpace->setStyleSheet("QLabel { color: #999900 }"); + } else { + ui->freeSpace->setStyleSheet(""); + } + ui->freeSpace->setText(freeString + "."); +} + void Intro::on_dataDirectory_textChanged(const QString &dataDirStr) { /* Disable OK button until check result comes in */ @@ -349,3 +350,20 @@ QString Intro::getPathToCheck() mutex.unlock(); return retval; } + +void Intro::UpdatePruneLabels(bool prune_checked) +{ + m_required_space_gb = m_blockchain_size_gb + m_chain_state_size_gb; + QString storageRequiresMsg = tr("At least %1 GB of data will be stored in this directory, and it will grow over time."); + if (prune_checked && m_prune_target_gb <= m_blockchain_size_gb) { + m_required_space_gb = m_prune_target_gb + m_chain_state_size_gb; + storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); + } + ui->lblExplanation3->setVisible(prune_checked); + ui->sizeWarningLabel->setText( + tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + + storageRequiresMsg.arg(m_required_space_gb) + " " + + tr("The wallet will also be stored in this directory.") + ); + this->adjustSize(); +} diff --git a/src/qt/intro.h b/src/qt/intro.h index aca7e71642..732393246e 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,7 @@ class Intro : public QDialog public: explicit Intro(QWidget *parent = nullptr, - uint64_t blockchain_size = 0, uint64_t chain_state_size = 0); + int64_t blockchain_size_gb = 0, int64_t chain_state_size_gb = 0); ~Intro(); QString getDataDirectory(); @@ -67,12 +67,18 @@ private: QMutex mutex; bool signalled; QString pathToCheck; - uint64_t m_blockchain_size; - uint64_t m_chain_state_size; + const int64_t m_blockchain_size_gb; + const int64_t m_chain_state_size_gb; + //! Total required space (in GB) depending on user choice (prune or not prune). + int64_t m_required_space_gb{0}; + uint64_t m_bytes_available{0}; + const int64_t m_prune_target_gb; void startThread(); void checkPath(const QString &dataDir); QString getPathToCheck(); + void UpdatePruneLabels(bool prune_checked); + void UpdateFreeSpaceLabel(); friend class FreespaceChecker; }; diff --git a/src/qt/macnotificationhandler.mm b/src/qt/macnotificationhandler.mm index a07079eece..b16042e946 100644 --- a/src/qt/macnotificationhandler.mm +++ b/src/qt/macnotificationhandler.mm @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2013 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 999c434d23..3dfd9e850e 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index efdd494d9f..6243a71c7d 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,7 @@ userClosed(false) setVisible(false); if (!enable_wallet) { ui->infoText->setVisible(false); - ui->infoTextStrong->setText(tr("Bitcoin Core is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.")); + ui->infoTextStrong->setText(tr("%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.").arg(PACKAGE_NAME)); } } diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index c075a89f94..076ec30b58 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index 5c039a939e..3a251e0573 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/networkstyle.h b/src/qt/networkstyle.h index 1367261325..a73e3e2625 100644 --- a/src/qt/networkstyle.h +++ b/src/qt/networkstyle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 9dc64bb23a..b9dea2f8bf 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index 8438f22bd7..4b610f74d7 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 3fffb8a288..8ee6c947e6 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -375,7 +375,7 @@ QValidator::State ProxyAddressValidator::validate(QString &input, int &pos) cons { Q_UNUSED(pos); // Validate the proxy - CService serv(LookupNumeric(input.toStdString().c_str(), DEFAULT_GUI_PROXY_PORT)); + CService serv(LookupNumeric(input.toStdString(), DEFAULT_GUI_PROXY_PORT)); proxyType addrProxy = proxyType(serv, true); if (addrProxy.IsValid()) return QValidator::Acceptable; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index d74d0dbfeb..977076c4c2 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -91,8 +91,8 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("bPrune")) settings.setValue("bPrune", false); if (!settings.contains("nPruneSize")) - settings.setValue("nPruneSize", 2); - SetPrune(settings.value("bPrune").toBool()); + settings.setValue("nPruneSize", DEFAULT_PRUNE_TARGET_GB); + SetPruneEnabled(settings.value("bPrune").toBool()); if (!settings.contains("nDatabaseCache")) settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache); @@ -236,13 +236,12 @@ static const QString GetDefaultProxyAddress() return QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST).arg(DEFAULT_GUI_PROXY_PORT); } -void OptionsModel::SetPrune(bool prune, bool force) +void OptionsModel::SetPruneEnabled(bool prune, bool force) { QSettings settings; settings.setValue("bPrune", prune); - // Convert prune size from GB to MiB: - const uint64_t nPruneSizeMiB = (settings.value("nPruneSize").toInt() * GB_BYTES) >> 20; - std::string prune_val = prune ? std::to_string(nPruneSizeMiB) : "0"; + const int64_t prune_target_mib = PruneGBtoMiB(settings.value("nPruneSize").toInt()); + std::string prune_val = prune ? std::to_string(prune_target_mib) : "0"; if (force) { m_node.forceSetArg("-prune", prune_val); return; @@ -252,6 +251,16 @@ void OptionsModel::SetPrune(bool prune, bool force) } } +void OptionsModel::SetPruneTargetGB(int prune_target_gb, bool force) +{ + const bool prune = prune_target_gb > 0; + if (prune) { + QSettings settings; + settings.setValue("nPruneSize", prune_target_gb); + } + SetPruneEnabled(prune, force); +} + // read QSettings values and return them QVariant OptionsModel::data(const QModelIndex & index, int role) const { diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 5791b47f28..b3260349e7 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,6 +6,7 @@ #define BITCOIN_QT_OPTIONSMODEL_H #include <amount.h> +#include <qt/guiconstants.h> #include <QAbstractListModel> @@ -16,6 +17,16 @@ class Node; extern const char *DEFAULT_GUI_PROXY_HOST; static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; +/** + * Convert configured prune target MiB to displayed GB. Round up to avoid underestimating max disk usage. + */ +static inline int PruneMiBtoGB(int64_t mib) { return (mib * 1024 * 1024 + GB_BYTES - 1) / GB_BYTES; } + +/** + * Convert displayed prune target GB to configured MiB. Round down so roundtrip GB -> MiB -> GB conversion is stable. + */ +static inline int64_t PruneGBtoMiB(int gb) { return gb * GB_BYTES / 1024 / 1024; } + /** Interface from Qt to configuration data structure for Bitcoin client. To Qt, the options are presented as a list with the different options laid out vertically. @@ -73,7 +84,8 @@ public: const QString& getOverriddenByCommandLine() { return strOverriddenByCommandLine; } /* Explicit setters */ - void SetPrune(bool prune, bool force = false); + void SetPruneEnabled(bool prune, bool force = false); + void SetPruneTargetGB(int prune_target_gb, bool force = false); /* Restart flag helper */ void setRestartRequired(bool fRequired); diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 07ffff0126..342c7cce31 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 6ad219ca2d..beca78a021 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 6d4ce4a7e4..aa9a7327ba 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 514ff35bcd..a497f58b16 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -1,10 +1,9 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/peertablemodel.h> -#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> @@ -100,10 +99,9 @@ public: } }; -PeerTableModel::PeerTableModel(interfaces::Node& node, ClientModel *parent) : +PeerTableModel::PeerTableModel(interfaces::Node& node, QObject* parent) : QAbstractTableModel(parent), m_node(node), - clientModel(parent), timer(nullptr) { columns << tr("NodeId") << tr("Node/Service") << tr("Ping") << tr("Sent") << tr("Received") << tr("User Agent"); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index b3f5dd7dbe..cf45c5a08f 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -13,7 +13,6 @@ #include <QAbstractTableModel> #include <QStringList> -class ClientModel; class PeerTablePriv; namespace interfaces { @@ -51,7 +50,7 @@ class PeerTableModel : public QAbstractTableModel Q_OBJECT public: - explicit PeerTableModel(interfaces::Node& node, ClientModel *parent = nullptr); + explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); const CNodeCombinedStats *getNodeStats(int idx); int getRowByNodeId(NodeId nodeid); @@ -83,7 +82,6 @@ public Q_SLOTS: private: interfaces::Node& m_node; - ClientModel *clientModel; QStringList columns; std::unique_ptr<PeerTablePriv> priv; QTimer *timer; diff --git a/src/qt/platformstyle.cpp b/src/qt/platformstyle.cpp index 08d692e44c..c6b80fd340 100644 --- a/src/qt/platformstyle.cpp +++ b/src/qt/platformstyle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/platformstyle.h b/src/qt/platformstyle.h index 635aec4c93..53632e56e2 100644 --- a/src/qt/platformstyle.h +++ b/src/qt/platformstyle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index 2332d52b9a..c816e1f8ed 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qrimagewidget.h b/src/qt/qrimagewidget.h index 2a219ac101..345bb64092 100644 --- a/src/qt/qrimagewidget.h +++ b/src/qt/qrimagewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qvaluecombobox.h b/src/qt/qvaluecombobox.h index 8892071fba..5cca515079 100644 --- a/src/qt/qvaluecombobox.h +++ b/src/qt/qvaluecombobox.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index de453cf743..16597e4758 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index 2674c9b953..b4fae7d78d 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index 1bb43ce6d1..40e3d5ffa8 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 18fa5f417f..7419297a96 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index f5085f7268..5e7f6acdc8 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/res/movies/makespinner.sh b/src/qt/res/movies/makespinner.sh index 3507837da9..4fa8dadf86 100755 --- a/src/qt/res/movies/makespinner.sh +++ b/src/qt/res/movies/makespinner.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 3dd64c5273..e1f783b0e5 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -905,12 +905,8 @@ void RPCConsole::on_lineEdit_returnPressed() cmdBeforeBrowsing = QString(); - WalletModel* wallet_model{nullptr}; #ifdef ENABLE_WALLET - const int wallet_index = ui->WalletSelector->currentIndex(); - if (wallet_index > 0) { - wallet_model = ui->WalletSelector->itemData(wallet_index).value<WalletModel*>(); - } + WalletModel* wallet_model = ui->WalletSelector->currentData().value<WalletModel*>(); if (m_last_wallet_model != wallet_model) { if (wallet_model) { @@ -1240,7 +1236,7 @@ void RPCConsole::unbanSelectedNode() QString strNode = nodes.at(i).data().toString(); CSubNet possibleSubnet; - LookupSubNet(strNode.toStdString().c_str(), possibleSubnet); + LookupSubNet(strNode.toStdString(), possibleSubnet); if (possibleSubnet.IsValid() && m_node.unban(possibleSubnet)) { clientModel->getBanTableModel()->refresh(); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index f1ea3e23e5..cc01aafb23 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index ccd8494613..86422c4030 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index 610dfbb85a..444dc79a2e 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsentry.h b/src/qt/sendcoinsentry.h index aa69d30f99..254cc186e2 100644 --- a/src/qt/sendcoinsentry.h +++ b/src/qt/sendcoinsentry.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 81e9d33a60..5f2836cc75 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 26c9fe7ad4..e19833019d 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 6e8d383847..0f082802cc 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <qt/test/addressbooktests.h> #include <qt/test/util.h> #include <test/util/setup_common.h> @@ -55,6 +59,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), WalletDatabase::CreateMock()); + wallet->SetupLegacyScriptPubKeyMan(); bool firstRun; wallet->LoadWallet(firstRun); diff --git a/src/qt/test/addressbooktests.h b/src/qt/test/addressbooktests.h index 9944750ec8..5de89c7592 100644 --- a/src/qt/test/addressbooktests.h +++ b/src/qt/test/addressbooktests.h @@ -1,3 +1,7 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_QT_TEST_ADDRESSBOOKTESTS_H #define BITCOIN_QT_TEST_ADDRESSBOOKTESTS_H diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 664826ecf2..14a75b23f3 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/apptests.h b/src/qt/test/apptests.h index 83bf56f1e4..d16c9fe487 100644 --- a/src/qt/test/apptests.h +++ b/src/qt/test/apptests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/compattests.cpp b/src/qt/test/compattests.cpp index cf86a5bc1e..c76dee5091 100644 --- a/src/qt/test/compattests.cpp +++ b/src/qt/test/compattests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 374fc9b59b..de1fbcb94c 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/rpcnestedtests.h b/src/qt/test/rpcnestedtests.h index 8789fe8373..0a00d1113a 100644 --- a/src/qt/test/rpcnestedtests.h +++ b/src/qt/test/rpcnestedtests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 243c10d7da..aefdcd2716 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -37,6 +37,8 @@ Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin); #endif #endif +const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; + // This is all you need to run all the tests int main(int argc, char *argv[]) { diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index ae2fb93bf7..e09f0ad77d 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <QApplication> #include <QMessageBox> #include <QPushButton> diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 377f07dcba..763847606a 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -1,3 +1,7 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_QT_TEST_UTIL_H #define BITCOIN_QT_TEST_UTIL_H diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index f6d2816ff8..c1a0f63f73 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2015-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <qt/test/wallettests.h> #include <qt/test/util.h> @@ -139,10 +143,9 @@ void TestGUI(interfaces::Node& node) bool firstRun; wallet->LoadWallet(firstRun); { - auto spk_man = wallet->GetLegacyScriptPubKeyMan(); + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); auto locked_chain = wallet->chain().lock(); - LOCK(wallet->cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); diff --git a/src/qt/test/wallettests.h b/src/qt/test/wallettests.h index 0a7b57a678..8ee40bf07f 100644 --- a/src/qt/test/wallettests.h +++ b/src/qt/test/wallettests.h @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_QT_TEST_WALLETTESTS_H #define BITCOIN_QT_TEST_WALLETTESTS_H diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 006007be63..757648f485 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index 48bd246b34..af5890ba24 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 9a93798aef..ece3a9cf48 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondescdialog.h b/src/qt/transactiondescdialog.h index 8fd3f3166a..74e34cde87 100644 --- a/src/qt/transactiondescdialog.h +++ b/src/qt/transactiondescdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 08ba030d65..a32d218fc9 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index fed55577ca..64e9c856db 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index cbc4ab49f5..3c638fb358 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 79347c371f..eca5656077 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 220e41b383..3101fb01c3 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,8 +10,6 @@ #include <qt/forms/ui_helpmessagedialog.h> -#include <qt/bitcoingui.h> - #include <clientversion.h> #include <init.h> #include <util/system.h> @@ -21,9 +19,10 @@ #include <QCloseEvent> #include <QLabel> +#include <QMainWindow> #include <QRegExp> -#include <QTextTable> #include <QTextCursor> +#include <QTextTable> #include <QVBoxLayout> /** "Help message" or "About" dialog box */ @@ -144,10 +143,9 @@ ShutdownWindow::ShutdownWindow(QWidget *parent, Qt::WindowFlags f): setLayout(layout); } -QWidget *ShutdownWindow::showShutdownWindow(BitcoinGUI *window) +QWidget* ShutdownWindow::showShutdownWindow(QMainWindow* window) { - if (!window) - return nullptr; + assert(window != nullptr); // Show a simple window indicating shutdown status QWidget *shutdownWindow = new ShutdownWindow(); diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index f1cedff282..833b86fd3e 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -6,9 +6,11 @@ #define BITCOIN_QT_UTILITYDIALOG_H #include <QDialog> -#include <QObject> +#include <QWidget> -class BitcoinGUI; +QT_BEGIN_NAMESPACE +class QMainWindow; +QT_END_NAMESPACE namespace interfaces { class Node; @@ -46,7 +48,7 @@ class ShutdownWindow : public QWidget public: explicit ShutdownWindow(QWidget *parent=nullptr, Qt::WindowFlags f=Qt::Widget); - static QWidget *showShutdownWindow(BitcoinGUI *window); + static QWidget* showShutdownWindow(QMainWindow* window); protected: void closeEvent(QCloseEvent *event); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 4b2b475883..656fffd067 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,14 +39,13 @@ void WalletFrame::setClientModel(ClientModel *_clientModel) this->clientModel = _clientModel; } -void WalletFrame::addWallet(WalletModel *walletModel) +bool WalletFrame::addWallet(WalletModel *walletModel) { - if (!gui || !clientModel || !walletModel) return; + if (!gui || !clientModel || !walletModel) return false; - if (mapWalletViews.count(walletModel) > 0) return; + if (mapWalletViews.count(walletModel) > 0) return false; WalletView *walletView = new WalletView(platformStyle, this); - walletView->setBitcoinGUI(gui); walletView->setClientModel(clientModel); walletView->setWalletModel(walletModel); walletView->showOutOfSyncWarning(bOutOfSync); @@ -62,6 +61,16 @@ void WalletFrame::addWallet(WalletModel *walletModel) mapWalletViews[walletModel] = walletView; connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); + connect(walletView, &WalletView::transactionClicked, gui, &BitcoinGUI::gotoHistoryPage); + connect(walletView, &WalletView::coinsSent, gui, &BitcoinGUI::gotoHistoryPage); + connect(walletView, &WalletView::message, [this](const QString& title, const QString& message, unsigned int style) { + gui->message(title, message, style); + }); + connect(walletView, &WalletView::encryptionStatusChanged, gui, &BitcoinGUI::updateWalletStatus); + connect(walletView, &WalletView::incomingTransaction, gui, &BitcoinGUI::incomingTransaction); + connect(walletView, &WalletView::hdEnabledStatusChanged, gui, &BitcoinGUI::updateWalletStatus); + + return true; } void WalletFrame::setCurrentWallet(WalletModel* wallet_model) diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 156653f47d..20fad08b0e 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -36,7 +36,7 @@ public: void setClientModel(ClientModel *clientModel); - void addWallet(WalletModel *walletModel); + bool addWallet(WalletModel *walletModel); void setCurrentWallet(WalletModel* wallet_model); void removeWallet(WalletModel* wallet_model); void removeAllWallets(); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index fb92e29f21..6c3a06f3a2 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -10,6 +10,7 @@ #include <qt/addresstablemodel.h> #include <qt/guiconstants.h> +#include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/paymentserver.h> #include <qt/recentrequeststablemodel.h> @@ -487,8 +488,10 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return false; } + const bool create_psbt = privateKeysDisabled(); + // allow a user based fee verification - QString questionString = tr("Do you want to increase the fee?"); + QString questionString = create_psbt ? tr("Do you want to draft a transaction with fee increase?") : tr("Do you want to increase the fee?"); questionString.append("<br />"); questionString.append("<table style=\"text-align: left;\">"); questionString.append("<tr><td>"); @@ -519,6 +522,23 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return false; } + // Short-circuit if we are returning a bumped transaction PSBT to clipboard + if (create_psbt) { + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + const TransactionError err = wallet().fillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + if (err != TransactionError::OK || complete) { + QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't draft transaction.")); + return false; + } + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + return true; + } + // sign bumped transaction if (!m_wallet->signBumpTransaction(mtx)) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't sign transaction.")); diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index eba95bd27c..25172e774c 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index 9e5d285e8c..f9a95362c8 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 8652827b59..bdcb82e06b 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,6 @@ #include <qt/addressbookpage.h> #include <qt/askpassphrasedialog.h> -#include <qt/bitcoingui.h> #include <qt/clientmodel.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -65,11 +64,13 @@ WalletView::WalletView(const PlatformStyle *_platformStyle, QWidget *parent): addWidget(receiveCoinsPage); addWidget(sendCoinsPage); + connect(overviewPage, &OverviewPage::transactionClicked, this, &WalletView::transactionClicked); // Clicking on a transaction on the overview pre-selects the transaction on the transaction history page connect(overviewPage, &OverviewPage::transactionClicked, transactionView, static_cast<void (TransactionView::*)(const QModelIndex&)>(&TransactionView::focusTransaction)); connect(overviewPage, &OverviewPage::outOfSyncWarningClicked, this, &WalletView::requestedSyncWarningInfo); + connect(sendCoinsPage, &SendCoinsDialog::coinsSent, this, &WalletView::coinsSent); // Highlight transaction after send connect(sendCoinsPage, &SendCoinsDialog::coinsSent, transactionView, static_cast<void (TransactionView::*)(const uint256&)>(&TransactionView::focusTransaction)); @@ -86,32 +87,6 @@ WalletView::~WalletView() { } -void WalletView::setBitcoinGUI(BitcoinGUI *gui) -{ - if (gui) - { - // Clicking on a transaction on the overview page simply sends you to transaction history page - connect(overviewPage, &OverviewPage::transactionClicked, gui, &BitcoinGUI::gotoHistoryPage); - - // Navigate to transaction history page after send - connect(sendCoinsPage, &SendCoinsDialog::coinsSent, gui, &BitcoinGUI::gotoHistoryPage); - - // Receive and report messages - connect(this, &WalletView::message, [gui](const QString &title, const QString &message, unsigned int style) { - gui->message(title, message, style); - }); - - // Pass through encryption status changed signals - connect(this, &WalletView::encryptionStatusChanged, gui, &BitcoinGUI::updateWalletStatus); - - // Pass through transaction notifications - connect(this, &WalletView::incomingTransaction, gui, &BitcoinGUI::incomingTransaction); - - // Connect HD enabled state signal - connect(this, &WalletView::hdEnabledStatusChanged, gui, &BitcoinGUI::updateWalletStatus); - } -} - void WalletView::setClientModel(ClientModel *_clientModel) { this->clientModel = _clientModel; diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 8d5a301cdb..86e46348be 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,6 @@ #include <QStackedWidget> -class BitcoinGUI; class ClientModel; class OverviewPage; class PlatformStyle; @@ -39,7 +38,6 @@ public: explicit WalletView(const PlatformStyle *platformStyle, QWidget *parent); ~WalletView(); - void setBitcoinGUI(BitcoinGUI *gui); /** Set the client model. The client model represents the part of the core that communicates with the P2P network, and is wallet-agnostic. */ @@ -115,6 +113,8 @@ public Q_SLOTS: void requestedSyncWarningInfo(); Q_SIGNALS: + void transactionClicked(); + void coinsSent(); /** Fired when a message should be reported to the user */ void message(const QString &title, const QString &message, unsigned int style); /** Encryption status of wallet changed */ diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index c6eb133cbd..386d593eea 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.cpp b/src/random.cpp index 99927fc5d2..f0082cf3e0 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.h b/src/random.h index e1b105168d..518a5cd3e3 100644 --- a/src/random.h +++ b/src/random.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rest.cpp b/src/rest.cpp index 55756ecfdf..0629557584 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index ccb3e39722..a02e5fae0e 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index dfca1697c1..2eaa3427eb 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 9f7f7837d3..69885546c8 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -102,7 +102,7 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } -static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) +static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; int nHeight = 0; @@ -116,7 +116,7 @@ static UniValue generateBlocks(const CScript& coinbase_script, int nGenerate, ui UniValue blockHashes(UniValue::VARR); while (nHeight < nHeightEnd && !ShutdownRequested()) { - std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(Params()).CreateNewBlock(coinbase_script)); + std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler(mempool, Params()).CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; @@ -179,9 +179,11 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); } + const CTxMemPool& mempool = EnsureMemPool(); + CHECK_NONFATAL(coinbase_script.size() == 1); - return generateBlocks(coinbase_script.at(0), num_blocks, max_tries); + return generateBlocks(mempool, coinbase_script.at(0), num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -215,9 +217,11 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address"); } + const CTxMemPool& mempool = EnsureMemPool(); + CScript coinbase_script = GetScriptForDestination(destination); - return generateBlocks(coinbase_script, nGenerate, nMaxTries); + return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); } static UniValue getmininginfo(const JSONRPCRequest& request) @@ -548,7 +552,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) // Create new block CScript scriptDummy = CScript() << OP_TRUE; - pblocktemplate = BlockAssembler(Params()).CreateNewBlock(scriptDummy); + pblocktemplate = BlockAssembler(mempool, Params()).CreateNewBlock(scriptDummy); if (!pblocktemplate) throw JSONRPCError(RPC_OUT_OF_MEMORY, "Out of memory"); @@ -904,7 +908,7 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) " lower buckets."}, }, RPCResult{ - "{\n" + "{ (json object) Results are returned for any horizon which tracks blocks up to the confirmation target\n" " \"short\" : { (json object, optional) estimate for short time horizon\n" " \"feerate\" : x.x, (numeric, optional) estimate fee rate in " + CURRENCY_UNIT + "/kB\n" " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" @@ -917,14 +921,22 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" " },\n" - " \"fail\" : { ... }, (json object, optional) information about the highest range of feerates to fail to meet the threshold\n" - " \"errors\": [ str... ] (json array of strings, optional) Errors encountered during processing\n" + " \"fail\" : { (json object, optional) information about the highest range of feerates to fail to meet the threshold\n" + " ...\n" + " },\n" + " \"errors\": [ (json array, optional) Errors encountered during processing\n" + " \"str\", (string)\n" + " ...\n" + " ],\n" + " },\n" + " \"medium\" : { (json object, optional) estimate for medium time horizon\n" + " ...\n" + " },\n" + " \"long\" : { (json object, optional) estimate for long time horizon\n" + " ...\n" " },\n" - " \"medium\" : { ... }, (json object, optional) estimate for medium time horizon\n" - " \"long\" : { ... } (json object) estimate for long time horizon\n" "}\n" "\n" - "Results are returned for any horizon which tracks blocks up to the confirmation target.\n" }, RPCExamples{ HelpExampleCli("estimaterawfee", "6 0.9") diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index d967d2bcca..56bd33b0ec 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 5e53fa5f5d..42aec08b45 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -83,6 +83,7 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) " \"addr\":\"host:port\", (string) The IP address and port of the peer\n" " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" + " \"mapped_as\":\"mapped_as\", (string) The AS in the BGP route to the peer used for diversifying peer selection\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"servicesnames\":[ (array) the services offered, in human-readable form\n" " \"SERVICE_NAME\", (string) the service name if it is recognised\n" @@ -152,6 +153,9 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("addrlocal", stats.addrLocal); if (stats.addrBind.IsValid()) obj.pushKV("addrbind", stats.addrBind.ToString()); + if (stats.m_mapped_as != 0) { + obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); + } obj.pushKV("services", strprintf("%016x", stats.nServices)); obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); obj.pushKV("relaytxes", stats.fRelayTxes); @@ -448,11 +452,11 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) "Returns an object containing various state info regarding P2P networking.\n", {}, RPCResult{ - "{\n" + "{ (json object)\n" " \"version\": xxxxx, (numeric) the server version\n" - " \"subversion\": \"/Satoshi:x.x.x/\", (string) the server subversion string\n" + " \"subversion\" : \"str\", (string) the server subversion string\n" " \"protocolversion\": xxxxx, (numeric) the protocol version\n" - " \"localservices\": \"xxxxxxxxxxxxxxxx\", (string) the services we offer to the network\n" + " \"localservices\" : \"hex\", (string) the services we offer to the network\n" " \"localservicesnames\": [ (array) the services we offer to the network, in human-readable form\n" " \"SERVICE_NAME\", (string) the service name\n" " ...\n" @@ -462,26 +466,26 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) " \"connections\": xxxxx, (numeric) the number of connections\n" " \"networkactive\": true|false, (bool) whether p2p networking is enabled\n" " \"networks\": [ (array) information per network\n" - " {\n" - " \"name\": \"xxx\", (string) network (ipv4, ipv6 or onion)\n" + " { (json object)\n" + " \"name\": \"str\", (string) network (ipv4, ipv6 or onion)\n" " \"limited\": true|false, (boolean) is the network limited using -onlynet?\n" " \"reachable\": true|false, (boolean) is the network reachable?\n" - " \"proxy\": \"host:port\" (string) the proxy that is used for this network, or empty if none\n" - " \"proxy_randomize_credentials\": true|false, (string) Whether randomized credentials are used\n" - " }\n" - " ,...\n" + " \"proxy\" : \"str\" (string) (\"host:port\") the proxy that is used for this network, or empty if none\n" + " \"proxy_randomize_credentials\" : true|false, (bool) Whether randomized credentials are used\n" + " },\n" + " ...\n" " ],\n" " \"relayfee\": x.xxxxxxxx, (numeric) minimum relay fee for transactions in " + CURRENCY_UNIT + "/kB\n" " \"incrementalfee\": x.xxxxxxxx, (numeric) minimum fee increment for mempool limiting or BIP 125 replacement in " + CURRENCY_UNIT + "/kB\n" " \"localaddresses\": [ (array) list of local addresses\n" - " {\n" - " \"address\": \"xxxx\", (string) network address\n" + " { (json object)\n" + " \"address\" : \"xxxx\", (string) network address\n" " \"port\": xxx, (numeric) network port\n" " \"score\": xxx (numeric) relative score\n" - " }\n" - " ,...\n" - " ]\n" - " \"warnings\": \"...\" (string) any network and blockchain warnings\n" + " },\n" + " ...\n" + " ],\n" + " \"warnings\" : \"str\", (string) any network and blockchain warnings\n" "}\n" }, RPCExamples{ @@ -562,11 +566,11 @@ static UniValue setban(const JSONRPCRequest& request) if (!isSubnet) { CNetAddr resolved; - LookupHost(request.params[0].get_str().c_str(), resolved, false); + LookupHost(request.params[0].get_str(), resolved, false); netAddr = resolved; } else - LookupSubNet(request.params[0].get_str().c_str(), subNet); + LookupSubNet(request.params[0].get_str(), subNet); if (! (isSubnet ? subNet.IsValid() : netAddr.IsValid()) ) throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Invalid IP/Subnet"); diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index ca779497b9..d1475f452d 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 5be7acce1c..972809a65b 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1079,7 +1079,12 @@ UniValue decodepsbt(const JSONRPCRequest& request) UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); - total_in += txout.nValue; + if (MoneyRange(txout.nValue) && MoneyRange(total_in + txout.nValue)) { + total_in += txout.nValue; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } UniValue o(UniValue::VOBJ); ScriptToUniv(txout.scriptPubKey, o, true); @@ -1089,7 +1094,13 @@ UniValue decodepsbt(const JSONRPCRequest& request) UniValue non_wit(UniValue::VOBJ); TxToUniv(*input.non_witness_utxo, uint256(), non_wit, false); in.pushKV("non_witness_utxo", non_wit); - total_in += input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; + CAmount utxo_val = input.non_witness_utxo->vout[psbtx.tx->vin[i].prevout.n].nValue; + if (MoneyRange(utxo_val) && MoneyRange(total_in + utxo_val)) { + total_in += utxo_val; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } } else { have_all_utxos = false; } @@ -1205,7 +1216,12 @@ UniValue decodepsbt(const JSONRPCRequest& request) outputs.push_back(out); // Fee calculation - output_value += psbtx.tx->vout[i].nValue; + if (MoneyRange(psbtx.tx->vout[i].nValue) && MoneyRange(output_value + psbtx.tx->vout[i].nValue)) { + output_value += psbtx.tx->vout[i].nValue; + } else { + // Hack to just not show fee later + have_all_utxos = false; + } } result.pushKV("outputs", outputs); if (have_all_utxos) { @@ -1276,11 +1292,10 @@ UniValue finalizepsbt(const JSONRPCRequest& request) " extract and return the complete transaction in normal network serialization instead of the PSBT."}, }, RPCResult{ - "{\n" - " \"psbt\" : \"value\", (string) The base64-encoded partially signed transaction if not extracted\n" - " \"hex\" : \"value\", (string) The hex-encoded network transaction if extracted\n" - " \"complete\" : true|false, (boolean) If the transaction has a complete set of signatures\n" - " ]\n" + "{ (json object)\n" + " \"psbt\" : \"str\", (string) The base64-encoded partially signed transaction if not extracted\n" + " \"hex\" : \"hex\", (string) The hex-encoded network transaction if extracted\n" + " \"complete\" : true|false, (boolean) If the transaction has a complete set of signatures\n" "}\n" }, RPCExamples{ diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 0b7712b83c..4750fd64ed 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,7 +28,7 @@ void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, /** * Parse a prevtxs UniValue array and get the map of coins from it * - * @param prevTxs Array of previous txns outputs that tx depends on but may not yet be in the block chain + * @param prevTxsUnival Array of previous txns outputs that tx depends on but may not yet be in the block chain * @param keystore A pointer to the temporary keystore if there is one * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call */ diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 91d3e1fca4..df8e687d82 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ #include <memory> // for unique_ptr #include <unordered_map> -static CCriticalSection cs_rpcWarmup; +static RecursiveMutex cs_rpcWarmup; static std::atomic<bool> g_rpc_running{false}; static bool fRPCInWarmup GUARDED_BY(cs_rpcWarmup) = true; static std::string rpcWarmupStatus GUARDED_BY(cs_rpcWarmup) = "RPC server started"; diff --git a/src/rpc/server.h b/src/rpc/server.h index be9c03bf6b..c91bf1f613 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 07a54335ac..927a3f3820 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/scheduler.h b/src/scheduler.h index 436f661c59..7080adf34c 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -98,7 +98,7 @@ class SingleThreadedSchedulerClient { private: CScheduler *m_pscheduler; - CCriticalSection m_cs_callbacks_pending; + RecursiveMutex m_cs_callbacks_pending; std::list<std::function<void ()>> m_callbacks_pending GUARDED_BY(m_cs_callbacks_pending); bool m_are_callbacks_running GUARDED_BY(m_cs_callbacks_pending) = false; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 32b388b7fa..773d6a55c4 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 5a1b55259a..a5a41d78dd 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -47,28 +47,28 @@ struct Descriptor { /** Expand a descriptor at a specified position. * - * @param[in] pos: The position at which to expand the descriptor. If IsRange() is false, this is ignored. - * @param[in] provider: The provider to query for private keys in case of hardened derivation. - * @param[out] output_scripts: The expanded scriptPubKeys. - * @param[out] out: Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). - * @param[out] cache: Cache data necessary to evaluate the descriptor at this point without access to private keys. + * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. + * @param[in] provider The provider to query for private keys in case of hardened derivation. + * @param[out] output_scripts The expanded scriptPubKeys. + * @param[out] out Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). + * @param[out] cache Cache data necessary to evaluate the descriptor at this point without access to private keys. */ virtual bool Expand(int pos, const SigningProvider& provider, std::vector<CScript>& output_scripts, FlatSigningProvider& out, std::vector<unsigned char>* cache = nullptr) const = 0; /** Expand a descriptor at a specified position using cached expansion data. * - * @param[in] pos: The position at which to expand the descriptor. If IsRange() is false, this is ignored. - * @param[in] cache: Cached expansion data. - * @param[out] output_scripts: The expanded scriptPubKeys. - * @param[out] out: Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). + * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. + * @param[in] cache Cached expansion data. + * @param[out] output_scripts The expanded scriptPubKeys. + * @param[out] out Scripts and public keys necessary for solving the expanded scriptPubKeys (may be equal to `provider`). */ virtual bool ExpandFromCache(int pos, const std::vector<unsigned char>& cache, std::vector<CScript>& output_scripts, FlatSigningProvider& out) const = 0; /** Expand the private key for a descriptor at a specified position, if possible. * - * @param[in] pos: The position at which to expand the descriptor. If IsRange() is false, this is ignored. - * @param[in] provider: The provider to query for the private keys. - * @param[out] out: Any private keys available for the specified `pos`. + * @param[in] pos The position at which to expand the descriptor. If IsRange() is false, this is ignored. + * @param[in] provider The provider to query for the private keys. + * @param[out] out Any private keys available for the specified `pos`. */ virtual void ExpandPrivate(int pos, const SigningProvider& provider, FlatSigningProvider& out) const = 0; }; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index ad833bc025..d0865d2793 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -1281,13 +1281,11 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn return ss.GetHash(); } - static const uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); - // Check for invalid use of SIGHASH_SINGLE if ((nHashType & 0x1f) == SIGHASH_SINGLE) { if (nIn >= txTo.vout.size()) { // nOut out of range - return one; + return UINT256_ONE(); } } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index d63d8b85b7..2b104a608c 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/script.cpp b/src/script/script.cpp index 0666a385d1..ae0de1d24e 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/script.h b/src/script/script.h index 6355b8a704..7aaa10b60b 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index eaf5363bd7..e7b6df3ce8 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 0ed92e8d5b..58eae3ce96 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -144,8 +144,13 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(valtype()); // workaround CHECKMULTISIG bug for (size_t i = 1; i < vSolutions.size() - 1; ++i) { CPubKey pubkey = CPubKey(vSolutions[i]); - if (ret.size() < required + 1 && CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) { - ret.push_back(std::move(sig)); + // We need to always call CreateSig in order to fill sigdata with all + // possible signatures that we can create. This will allow further PSBT + // processing to work as it needs all possible signature and pubkey pairs + if (CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) { + if (ret.size() < required + 1) { + ret.push_back(std::move(sig)); + } } } bool ok = ret.size() == required + 1; diff --git a/src/script/sign.h b/src/script/sign.h index 4c2403f83f..033c9ba19e 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/signingprovider.h b/src/script/signingprovider.h index c40fecac5c..6ad20480a7 100644 --- a/src/script/signingprovider.h +++ b/src/script/signingprovider.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -72,7 +72,7 @@ protected: void ImplicitlyLearnRelatedKeyScripts(const CPubKey& pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); public: - mutable CCriticalSection cs_KeyStore; + mutable RecursiveMutex cs_KeyStore; virtual bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey); virtual bool AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 144bdcff98..7d89a336fb 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/standard.h b/src/script/standard.h index 6db28dbc2d..49a45f3eba 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -47,7 +47,7 @@ extern unsigned nMaxDatacarrierBytes; * but in the future other flags may be added, such as a soft-fork to enforce * strict DER encoding. * - * Failing one of these tests may trigger a DoS ban - see CheckInputs() for + * Failing one of these tests may trigger a DoS ban - see CheckInputScripts() for * details. */ static const unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS = SCRIPT_VERIFY_P2SH; @@ -80,9 +80,14 @@ struct PKHash : public uint160 using uint160::uint160; }; +struct WitnessV0KeyHash; struct ScriptHash : public uint160 { ScriptHash() : uint160() {} + // These don't do what you'd expect. + // Use ScriptHash(GetScriptForDestination(...)) instead. + explicit ScriptHash(const WitnessV0KeyHash& hash) = delete; + explicit ScriptHash(const PKHash& hash) = delete; explicit ScriptHash(const uint160& hash) : uint160(hash) {} explicit ScriptHash(const CScript& script); using uint160::uint160; diff --git a/src/serialize.h b/src/serialize.h index ef270dbbe3..7fa669ebdb 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -199,6 +199,52 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; } SerializationOp(s, CSerActionUnserialize()); \ } +/** + * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). + * + * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic + * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when + * deserializing). + * + * Example use: + * struct FooFormatter { + * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } + * } + * would define a class FooFormatter that defines a serialization of Class objects consisting + * of serializing its val1 member using the default serialization, and its val2 member using + * VARINT serialization. That FooFormatter can then be used in statements like + * READWRITE(Using<FooFormatter>(obj.bla)). + */ +#define FORMATTER_METHODS(cls, obj) \ + template<typename Stream> \ + static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \ + template<typename Stream> \ + static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \ + template<typename Stream, typename Type, typename Operation> \ + static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ + +/** + * Implement the Serialize and Unserialize methods by delegating to a single templated + * static method that takes the to-be-(de)serialized object as a parameter. This approach + * has the advantage that the constness of the object becomes a template parameter, and + * thus allows a single implementation that sees the object as const for serializing + * and non-const for deserializing, without casts. + */ +#define SERIALIZE_METHODS(cls, obj) \ + template<typename Stream> \ + void Serialize(Stream& s) const \ + { \ + static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \ + Ser(s, *this); \ + } \ + template<typename Stream> \ + void Unserialize(Stream& s) \ + { \ + static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \ + Unser(s, *this); \ + } \ + FORMATTER_METHODS(cls, obj) + #ifndef CHAR_EQUALS_INT8 template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char #endif @@ -418,26 +464,48 @@ I ReadVarInt(Stream& is) } } -#define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) -#define COMPACTSIZE(obj) CCompactSize(REF(obj)) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) - -template<VarIntMode Mode, typename I> -class CVarInt +/** Simple wrapper class to serialize objects using a formatter; used by Using(). */ +template<typename Formatter, typename T> +class Wrapper { + static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T"); protected: - I &n; + T m_object; public: - explicit CVarInt(I& nIn) : n(nIn) { } + explicit Wrapper(T obj) : m_object(obj) {} + template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); } + template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); } +}; - template<typename Stream> - void Serialize(Stream &s) const { - WriteVarInt<Stream,Mode,I>(s, n); +/** Cause serialization/deserialization of an object to be done using a specified formatter class. + * + * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for + * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside + * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object). + * + * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is + * const during serialization, and non-const during deserialization, which maintains const + * correctness. + */ +template<typename Formatter, typename T> +static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); } + +#define VARINT(obj, ...) Using<VarIntFormatter<__VA_ARGS__>>(obj) +#define COMPACTSIZE(obj) CCompactSize(REF(obj)) +#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) + +/** Serialization wrapper class for integers in VarInt format. */ +template<VarIntMode Mode=VarIntMode::DEFAULT> +struct VarIntFormatter +{ + template<typename Stream, typename I> void Ser(Stream &s, I v) + { + WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v); } - template<typename Stream> - void Unserialize(Stream& s) { - n = ReadVarInt<Stream,Mode,I>(s); + template<typename Stream, typename I> void Unser(Stream& s, I& v) + { + v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s); } }; @@ -522,9 +590,6 @@ public: } }; -template<VarIntMode Mode=VarIntMode::DEFAULT, typename I> -CVarInt<Mode, I> WrapVarInt(I& n) { return CVarInt<Mode, I>{n}; } - template<typename I> BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); } diff --git a/src/streams.h b/src/streams.h index b598dc1aeb..e1d1b0eab2 100644 --- a/src/streams.h +++ b/src/streams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 57f5b1f733..0e31ad3ce3 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/cleanse.cpp b/src/support/cleanse.cpp index ecb00510f7..a8ddcd793f 100644 --- a/src/support/cleanse.cpp +++ b/src/support/cleanse.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/cleanse.h b/src/support/cleanse.h index b03520315d..8c1210a114 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 85e3351e72..6980b6c0da 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index b420c909fc..de668f0773 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.cpp b/src/sync.cpp index 257093fad1..924e7b5bb0 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,7 +75,7 @@ typedef std::set<std::pair<void*, void*> > InvLockOrders; struct LockData { // Very ugly hack: as the global constructs and destructors run single // threaded, we use this boolean to know whether LockData still exists, - // as DeleteLock can get called by global CCriticalSection destructors + // as DeleteLock can get called by global RecursiveMutex destructors // after LockData disappears. bool available; LockData() : available(true) {} diff --git a/src/sync.h b/src/sync.h index 8ff6173142..0cdbb59c70 100644 --- a/src/sync.h +++ b/src/sync.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -106,7 +106,6 @@ public: * TODO: We should move away from using the recursive lock by default. */ using RecursiveMutex = AnnotatedMixin<std::recursive_mutex>; -typedef AnnotatedMixin<std::recursive_mutex> CCriticalSection; /** Wrapped mutex: supports waiting but not recursive locking */ typedef AnnotatedMixin<std::mutex> Mutex; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index c034216bc1..07cebeb35a 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -5,6 +5,8 @@ #include <test/util/setup_common.h> #include <string> #include <boost/test/unit_test.hpp> +#include <util/asmap.h> +#include <test/data/asmap.raw.h> #include <hash.h> #include <netbase.h> @@ -12,13 +14,18 @@ class CAddrManTest : public CAddrMan { +private: + bool deterministic; public: - explicit CAddrManTest(bool makeDeterministic = true) + explicit CAddrManTest(bool makeDeterministic = true, + std::vector<bool> asmap = std::vector<bool>()) { if (makeDeterministic) { // Set addrman addr placement to be deterministic. MakeDeterministic(); } + deterministic = makeDeterministic; + m_asmap = asmap; } //! Ensure that bucket placement is always the same for testing purposes. @@ -46,6 +53,21 @@ public: CAddrMan::Delete(nId); } + // Used to test deserialization + std::pair<int, int> GetBucketAndEntry(const CAddress& addr) + { + LOCK(cs); + int nId = mapAddr[addr]; + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { + if (nId == vvNew[bucket][entry]) { + return std::pair<int, int>(bucket, entry); + } + } + } + return std::pair<int, int>(-1, -1); + } + // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(CService& addr) { @@ -57,32 +79,45 @@ public: int64_t nLastTry = GetAdjustedTime()-61; Attempt(addr, count_failure, nLastTry); } + + void Clear() + { + CAddrMan::Clear(); + if (deterministic) { + nKey.SetNull(); + insecure_rand = FastRandomContext(true); + } + } + }; -static CNetAddr ResolveIP(const char* ip) +static CNetAddr ResolveIP(const std::string& ip) { CNetAddr addr; BOOST_CHECK_MESSAGE(LookupHost(ip, addr, false), strprintf("failed to resolve: %s", ip)); return addr; } -static CNetAddr ResolveIP(std::string ip) -{ - return ResolveIP(ip.c_str()); -} - -static CService ResolveService(const char* ip, int port = 0) +static CService ResolveService(const std::string& ip, const int port = 0) { CService serv; BOOST_CHECK_MESSAGE(Lookup(ip, serv, port, false), strprintf("failed to resolve: %s:%i", ip, port)); return serv; } -static CService ResolveService(std::string ip, int port = 0) -{ - return ResolveService(ip.c_str(), port); + +static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) { + std::vector<bool> result(vector_size); + for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { + unsigned char cur_byte = source[byte_i]; + for (int bit_i = 0; bit_i < 8; ++bit_i) { + result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1; + } + } + return result; } + BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) @@ -409,7 +444,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } -BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { CAddrManTest addrman; @@ -424,30 +459,31 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info2.GetTriedBucket(nKey1)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetTriedBucket(nKey1); + int bucket = infoi.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same group (\16 prefix for IPv4) should - // never get more than 8 buckets + // Test: IP addresses in the same /16 prefix should + // never get more than 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 8U); buckets.clear(); @@ -455,15 +491,15 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), ResolveIP("250." + std::to_string(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1); + int bucket = infoj.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different groups should map to more than - // 8 buckets. + // Test: IP addresses in the different /16 prefix should map to more than + // 8 buckets with legacy grouping BOOST_CHECK_EQUAL(buckets.size(), 160U); } -BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { CAddrManTest addrman; @@ -477,25 +513,27 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector<bool> asmap; // use /16 + // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetNewBucket(nKey1); + int bucket = infoi.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } // Test: IP addresses in the same group (\16 prefix for IPv4) should @@ -508,10 +546,10 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to NO MORE // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -520,14 +558,226 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + std::to_string(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to MORE // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } +// The following three test cases use asmap.raw +// We use an artificial minimal mock mapping +// 250.0.0.0/8 AS1000 +// 101.1.0.0/16 AS1 +// 101.2.0.0/16 AS2 +// 101.3.0.0/16 AS3 +// 101.4.0.0/16 AS4 +// 101.5.0.0/16 AS5 +// 101.6.0.0/16 AS6 +// 101.7.0.0/16 AS7 +// 101.8.0.0/16 AS8 +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.1.1"); + + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + + // Test: Two addresses with same IP but different ports can map to + // different buckets because they have different keys. + CAddrInfo info2 = CAddrInfo(addr2, source1); + + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("101." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() > 8); + + buckets.clear(); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix MAY NOT map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() == 8); +} + +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.2.1"); + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + + // Test: Ports should not affect bucket placement in the addr + CAddrInfo info2 = CAddrInfo(addr2, source1); + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + + std::set<int> buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same /16 prefix + // usually map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); + + buckets.clear(); + for (int j = 0; j < 4 * 255; j++) { + CAddrInfo infoj = CAddrInfo(CAddress( + ResolveService( + std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), + ResolveIP("251.4.1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same source /16 prefix should not map to more + // than 64 buckets. + BOOST_CHECK(buckets.size() <= 64); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes usually map to MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() > 1); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() == 1); + +} + +BOOST_AUTO_TEST_CASE(addrman_serialization) +{ + std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + + CAddrManTest addrman_asmap1(true, asmap1); + CAddrManTest addrman_asmap1_dup(true, asmap1); + CAddrManTest addrman_noasmap; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CNetAddr default_source; + + + addrman_asmap1.Add(addr, default_source); + + stream << addrman_asmap1; + // serizalizing/deserializing addrman with the same asmap + stream >> addrman_asmap1_dup; + + std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); + std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + + BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + + // deserializing asmaped peers.dat to non-asmaped addrman + stream << addrman_asmap1; + stream >> addrman_noasmap; + std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_noasmap.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + + // deserializing non-asmaped peers.dat to asmaped addrman + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + addrman_noasmap.Add(addr, default_source); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + // used to map to different buckets, now maps to the same bucket. + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); + addrman_noasmap.Add(addr, default_source); + addrman_noasmap.Add(addr2, default_source); + std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); +} + BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index 29935c09f8..3b4c480f72 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <boost/test/unit_test.hpp> #include <stdlib.h> diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 2b616f4c2b..79e18cd2c0 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -18,6 +18,11 @@ BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) +struct BuildChainTestingSetup : public TestChain100Setup { + CBlock CreateBlock(const CBlockIndex* prev, const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey); + bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, size_t length, std::vector<std::shared_ptr<CBlock>>& chain); +}; + static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, uint256& last_header) { @@ -52,12 +57,12 @@ static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex return true; } -static CBlock CreateBlock(const CBlockIndex* prev, - const std::vector<CMutableTransaction>& txns, - const CScript& scriptPubKey) +CBlock BuildChainTestingSetup::CreateBlock(const CBlockIndex* prev, + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; block.hashPrevBlock = prev->GetBlockHash(); block.nTime = prev->nTime + 1; @@ -76,8 +81,10 @@ static CBlock CreateBlock(const CBlockIndex* prev, return block; } -static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, - size_t length, std::vector<std::shared_ptr<CBlock>>& chain) +bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, + const CScript& coinbase_script_pub_key, + size_t length, + std::vector<std::shared_ptr<CBlock>>& chain) { std::vector<CMutableTransaction> no_txns; @@ -95,7 +102,7 @@ static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script return true; } -BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true); diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 119d4b3295..6be24c0845 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -7,6 +7,7 @@ #include <test/util/setup_common.h> #include <random.h> #include <thread> +#include <deque> /** Test Suite for CuckooCache * diff --git a/src/test/data/asmap.raw b/src/test/data/asmap.raw Binary files differnew file mode 100644 index 0000000000..3dcf1f3940 --- /dev/null +++ b/src/test/data/asmap.raw diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 632151793e..2c2b3035e3 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -51,7 +51,7 @@ struct COrphanTx { NodeId fromPeer; int64_t nTimeExpire; }; -extern CCriticalSection g_cs_orphans; +extern RecursiveMutex g_cs_orphans; extern std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); static CService ip(uint32_t i) diff --git a/src/test/fuzz/FuzzedDataProvider.h b/src/test/fuzz/FuzzedDataProvider.h index 1b5b4bb012..3e069eba69 100644 --- a/src/test/fuzz/FuzzedDataProvider.h +++ b/src/test/fuzz/FuzzedDataProvider.h @@ -13,11 +13,10 @@ #ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ #define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_ -#include <limits.h> -#include <stddef.h> -#include <stdint.h> - #include <algorithm> +#include <climits> +#include <cstddef> +#include <cstdint> #include <cstring> #include <initializer_list> #include <string> @@ -25,8 +24,10 @@ #include <utility> #include <vector> +// In addition to the comments below, the API is also briefly documented at +// https://github.com/google/fuzzing/blob/master/docs/split-inputs.md#fuzzed-data-provider class FuzzedDataProvider { -public: + public: // |data| is an array of length |size| that the FuzzedDataProvider wraps to // provide more granular access. |data| must outlive the FuzzedDataProvider. FuzzedDataProvider(const uint8_t *data, size_t size) @@ -143,9 +144,9 @@ public: return ConsumeBytes<T>(remaining_bytes_); } + // Returns a std::string containing all remaining bytes of the input data. // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string // object. - // Returns a std::vector containing all remaining bytes of the input data. std::string ConsumeRemainingBytesAsString() { return ConsumeBytesAsString(remaining_bytes_); } @@ -161,7 +162,7 @@ public: // Reads one byte and returns a bool, or false when no data remains. bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); } - // Returns a copy of a value selected from a fixed-size |array|. + // Returns a copy of the value selected from the given fixed-size |array|. template <typename T, size_t size> T PickValueInArray(const T (&array)[size]) { static_assert(size > 0, "The array must be non empty."); @@ -170,11 +171,14 @@ public: template <typename T> T PickValueInArray(std::initializer_list<const T> list) { - // static_assert(list.size() > 0, "The array must be non empty."); + // TODO(Dor1s): switch to static_assert once C++14 is allowed. + if (!list.size()) + abort(); + return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1)); } - // Return an enum value. The enum must start at 0 and be contiguous. It must + // Returns an enum value. The enum must start at 0 and be contiguous. It must // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as: // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue }; template <typename T> T ConsumeEnum() { @@ -183,10 +187,60 @@ public: 0, static_cast<uint32_t>(T::kMaxValue))); } + // Returns a floating point number in the range [0.0, 1.0]. If there's no + // input data left, always returns 0. + template <typename T> T ConsumeProbability() { + static_assert(std::is_floating_point<T>::value, + "A floating point type is required."); + + // Use different integral types for different floating point types in order + // to provide better density of the resulting values. + using IntegralType = + typename std::conditional<(sizeof(T) <= sizeof(uint32_t)), uint32_t, + uint64_t>::type; + + T result = static_cast<T>(ConsumeIntegral<IntegralType>()); + result /= static_cast<T>(std::numeric_limits<IntegralType>::max()); + return result; + } + + // Returns a floating point value in the range [Type's lowest, Type's max] by + // consuming bytes from the input data. If there's no input data left, always + // returns approximately 0. + template <typename T> T ConsumeFloatingPoint() { + return ConsumeFloatingPointInRange<T>(std::numeric_limits<T>::lowest(), + std::numeric_limits<T>::max()); + } + + // Returns a floating point value in the given range by consuming bytes from + // the input data. If there's no input data left, returns |min|. Note that + // |min| must be less than or equal to |max|. + template <typename T> T ConsumeFloatingPointInRange(T min, T max) { + if (min > max) + abort(); + + T range = .0; + T result = min; + constexpr T zero(.0); + if (max > zero && min < zero && max > min + std::numeric_limits<T>::max()) { + // The diff |max - min| would overflow the given floating point type. Use + // the half of the diff as the range and consume a bool to decide whether + // the result is in the first of the second part of the diff. + range = (max / 2.0) - (min / 2.0); + if (ConsumeBool()) { + result += range; + } + } else { + range = max - min; + } + + return result + range * ConsumeProbability<T>(); + } + // Reports the remaining bytes available for fuzzed input. size_t remaining_bytes() { return remaining_bytes_; } -private: + private: FuzzedDataProvider(const FuzzedDataProvider &) = delete; FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete; @@ -209,6 +263,12 @@ private: // which seems to be a natural choice for other implementations as well. // To increase the odds even more, we also call |shrink_to_fit| below. std::vector<T> result(size); + if (size == 0) { + if (num_bytes_to_consume != 0) + abort(); + return result; + } + std::memcpy(result.data(), data_ptr_, num_bytes_to_consume); Advance(num_bytes_to_consume); @@ -230,9 +290,9 @@ private: // Avoid using implementation-defined unsigned to signer conversions. // To learn more, see https://stackoverflow.com/questions/13150449. - if (value <= std::numeric_limits<TS>::max()) + if (value <= std::numeric_limits<TS>::max()) { return static_cast<TS>(value); - else { + } else { constexpr auto TS_min = std::numeric_limits<TS>::min(); return TS_min + static_cast<char>(value - TS_min); } diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp new file mode 100644 index 0000000000..7f3eef79a1 --- /dev/null +++ b/src/test/fuzz/asmap.cpp @@ -0,0 +1,28 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <netaddress.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> + +#include <cstdint> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const Network network = fuzzed_data_provider.PickValueInArray({NET_IPV4, NET_IPV6}); + if (fuzzed_data_provider.remaining_bytes() < 16) { + return; + } + CNetAddr net_addr; + net_addr.SetRaw(network, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data()); + std::vector<bool> asmap; + for (const char cur_byte : fuzzed_data_provider.ConsumeRemainingBytes<char>()) { + for (int bit = 0; bit < 8; ++bit) { + asmap.push_back((cur_byte >> bit) & 1); + } + } + (void)net_addr.GetMappedAS(asmap); +} diff --git a/src/test/fuzz/decode_tx.cpp b/src/test/fuzz/decode_tx.cpp new file mode 100644 index 0000000000..09c4ff05df --- /dev/null +++ b/src/test/fuzz/decode_tx.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <primitives/transaction.h> +#include <test/fuzz/fuzz.h> +#include <util/strencodings.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + const std::string tx_hex = HexStr(std::string{buffer.begin(), buffer.end()}); + CMutableTransaction mtx; + const bool result_none = DecodeHexTx(mtx, tx_hex, false, false); + const bool result_try_witness = DecodeHexTx(mtx, tx_hex, false, true); + const bool result_try_witness_and_maybe_no_witness = DecodeHexTx(mtx, tx_hex, true, true); + const bool result_try_no_witness = DecodeHexTx(mtx, tx_hex, true, false); + assert(!result_none); + if (result_try_witness_and_maybe_no_witness) { + assert(result_try_no_witness || result_try_witness); + } + // if (result_try_no_witness) { // Uncomment when https://github.com/bitcoin/bitcoin/pull/17775 is merged + if (result_try_witness) { // Remove stop-gap when https://github.com/bitcoin/bitcoin/pull/17775 is merged + assert(result_try_witness_and_maybe_no_witness); + } +} diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index bd05283b78..f06f339b9d 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -206,7 +206,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) DeserializeFromFuzzingInput(buffer, dbi); #elif TXOUTCOMPRESSOR_DESERIALIZE CTxOut to; - CTxOutCompressor toc(to); + auto toc = Using<TxOutCompression>(to); DeserializeFromFuzzingInput(buffer, toc); #elif BLOCKTRANSACTIONS_DESERIALIZE BlockTransactions bt; diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index da4e623e98..a085e36911 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -4,10 +4,15 @@ #include <test/fuzz/fuzz.h> +#include <test/util/setup_common.h> + #include <cstdint> #include <unistd.h> #include <vector> +const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; + +#if defined(__AFL_COMPILER) static bool read_stdin(std::vector<uint8_t>& data) { uint8_t buffer[1024]; @@ -19,6 +24,7 @@ static bool read_stdin(std::vector<uint8_t>& data) } return length == 0; } +#endif // Default initialization: Override using a non-weak initialize(). __attribute__((weak)) void initialize() @@ -40,9 +46,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) return 0; } -// Declare main(...) "weak" to allow for libFuzzer linking. libFuzzer provides -// the main(...) function. -__attribute__((weak)) int main(int argc, char** argv) +// Generally, the fuzzer will provide main(), except for AFL +#if defined(__AFL_COMPILER) +int main(int argc, char** argv) { initialize(); #ifdef __AFL_INIT @@ -70,3 +76,4 @@ __attribute__((weak)) int main(int argc, char** argv) #endif return 0; } +#endif diff --git a/src/test/fuzz/strprintf.cpp b/src/test/fuzz/strprintf.cpp new file mode 100644 index 0000000000..0de21f0e7c --- /dev/null +++ b/src/test/fuzz/strprintf.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <tinyformat.h> +#include <util/strencodings.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::string format_string = fuzzed_data_provider.ConsumeRandomLengthString(64); + + const int digits_in_format_specifier = std::count_if(format_string.begin(), format_string.end(), IsDigit); + + // Avoid triggering the following crash bug: + // * strprintf("%987654321000000:", 1); + // + // Avoid triggering the following OOM bug: + // * strprintf("%.222222200000000$", 1.1); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + if (format_string.find("%") != std::string::npos && digits_in_format_specifier >= 7) { + return; + } + + // Avoid triggering the following crash bug: + // * strprintf("%1$*1$*", -11111111); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + if (format_string.find("%") != std::string::npos && format_string.find("$") != std::string::npos && format_string.find("*") != std::string::npos && digits_in_format_specifier > 0) { + return; + } + + // Avoid triggering the following crash bug: + // * strprintf("%.1s", (char*)nullptr); + // + // (void)strprintf(format_string, (char*)nullptr); + // + // Upstream bug report: https://github.com/c42f/tinyformat/issues/70 + + try { + (void)strprintf(format_string, (signed char*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (unsigned char*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (void*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (bool*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (float*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (double*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int16_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint16_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int32_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint32_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (int64_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + try { + (void)strprintf(format_string, (uint64_t*)nullptr); + } catch (const tinyformat::format_error&) { + } + + try { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 13)) { + case 0: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32)); + break; + case 1: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeRandomLengthString(32).c_str()); + break; + case 2: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<signed char>()); + break; + case 3: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<unsigned char>()); + break; + case 4: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<char>()); + break; + case 5: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeBool()); + break; + case 6: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<float>()); + break; + case 7: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeFloatingPoint<double>()); + break; + case 8: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int16_t>()); + break; + case 9: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint16_t>()); + break; + case 10: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int32_t>()); + break; + case 11: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + break; + case 12: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<int64_t>()); + break; + case 13: + (void)strprintf(format_string, fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + break; + default: + assert(false); + } + } catch (const tinyformat::format_error&) { + } +} diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 4c64d8c833..10fb05ca8a 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -183,4 +183,32 @@ BOOST_AUTO_TEST_CASE(boolargno) BOOST_CHECK(gArgs.GetBoolArg("-foo", false)); } +BOOST_AUTO_TEST_CASE(logargs) +{ + const auto okaylog_bool = std::make_pair("-okaylog-bool", ArgsManager::ALLOW_BOOL); + const auto okaylog_negbool = std::make_pair("-okaylog-negbool", ArgsManager::ALLOW_BOOL); + const auto okaylog = std::make_pair("-okaylog", ArgsManager::ALLOW_ANY); + const auto dontlog = std::make_pair("-dontlog", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE); + SetupArgs({okaylog_bool, okaylog_negbool, okaylog, dontlog}); + ResetArgs("-okaylog-bool -nookaylog-negbool -okaylog=public -dontlog=private"); + + // Everything logged to debug.log will also append to str + std::string str; + auto print_connection = LogInstance().PushBackCallback( + [&str](const std::string& s) { + str += s; + }); + + // Log the arguments + gArgs.LogArgs(); + + LogInstance().DeleteCallback(print_connection); + // Check that what should appear does, and what shouldn't doesn't. + BOOST_CHECK(str.find("Command-line arg: okaylog-bool=\"\"") != std::string::npos); + BOOST_CHECK(str.find("Command-line arg: okaylog-negbool=false") != std::string::npos); + BOOST_CHECK(str.find("Command-line arg: okaylog=\"public\"") != std::string::npos); + BOOST_CHECK(str.find("dontlog=****") != std::string::npos); + BOOST_CHECK(str.find("private") == std::string::npos); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/main.cpp b/src/test/main.cpp index ff3f36b561..e6529949e2 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -2,6 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +/** + * See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/link_references/link_boost_test_module_macro.html + */ #define BOOST_TEST_MODULE Bitcoin Core Test Suite #include <boost/test/unit_test.hpp> + +#include <test/util/setup_common.h> + +/** Redirect debug log to boost log */ +const std::function<void(const std::string&)> G_TEST_LOG_FUN = [](const std::string& s) { + if (s.back() == '\n') { + // boost will insert the new line + BOOST_TEST_MESSAGE(s.substr(0, s.size() - 1)); + } else { + BOOST_TEST_MESSAGE(s); + } +}; diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index d79a598bf1..9f3ca87206 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -30,6 +30,7 @@ struct MinerTestingSetup : public TestingSetup { { return CheckSequenceLocks(*m_node.mempool, tx, flags); } + BlockAssembler AssemblerForTest(const CChainParams& params); }; } // namespace miner_tests @@ -48,16 +49,16 @@ private: static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); -static BlockAssembler AssemblerForTest(const CChainParams& params) { +BlockAssembler MinerTestingSetup::AssemblerForTest(const CChainParams& params) +{ BlockAssembler::Options options; options.nBlockMaxWeight = MAX_BLOCK_WEIGHT; options.blockMinFeeRate = blockMinFeeRate; - return BlockAssembler(params, options); + return BlockAssembler(*m_node.mempool, params, options); } -static -struct { +constexpr static struct { unsigned char extranonce; unsigned int nonce; } blockinfo[] = { @@ -225,7 +226,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); // We can't make transactions until we have inputs - // Therefore, load 100 blocks :) + // Therefore, load 110 blocks :) + static_assert(sizeof(blockinfo) / sizeof(*blockinfo) == 110, "Should have 110 blocks to import"); int baseheight = 0; std::vector<CTransactionRef> txFirst; for (unsigned int i = 0; i < sizeof(blockinfo)/sizeof(*blockinfo); ++i) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index daf7fea6ad..cb1ef5dcf3 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -99,6 +99,8 @@ BOOST_AUTO_TEST_CASE(caddrdb_read) BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); + BOOST_CHECK(Lookup(std::string("250.7.3.3", 9), addr3, 9999, false)); + BOOST_CHECK(!Lookup(std::string("250.7.3.3\0example.com", 21), addr3, 9999, false)); // Add three addresses to new table. CService source; diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 481dedc356..9730b40580 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -13,21 +13,21 @@ BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) -static CNetAddr ResolveIP(const char* ip) +static CNetAddr ResolveIP(const std::string& ip) { CNetAddr addr; LookupHost(ip, addr, false); return addr; } -static CSubNet ResolveSubNet(const char* subnet) +static CSubNet ResolveSubNet(const std::string& subnet) { CSubNet ret; LookupSubNet(subnet, ret); return ret; } -static CNetAddr CreateInternal(const char* host) +static CNetAddr CreateInternal(const std::string& host) { CNetAddr addr; addr.SetInternal(host); @@ -105,7 +105,7 @@ BOOST_AUTO_TEST_CASE(netbase_splithost) bool static TestParse(std::string src, std::string canon) { - CService addr(LookupNumeric(src.c_str(), 65535)); + CService addr(LookupNumeric(src, 65535)); return canon == addr.ToString(); } @@ -127,7 +127,6 @@ BOOST_AUTO_TEST_CASE(netbase_lookupnumeric) BOOST_AUTO_TEST_CASE(onioncat_test) { - // values from https://web.archive.org/web/20121122003543/http://www.cypherpunk.at/onioncat/wiki/OnionCat CNetAddr addr1(ResolveIP("5wyqrzbvrdsumnok.onion")); CNetAddr addr2(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca")); @@ -287,23 +286,23 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup() == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup() == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + std::vector<bool> asmap; // use /16 + BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // Local -> !Routable() + BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // !Valid -> !Routable() + BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() + BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() + BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_ONION, 239})); // Tor + BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector<unsigned char> internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup() == internal_group); + BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_parsenetwork) @@ -402,4 +401,22 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_CHECK(std::find(strings.begin(), strings.end(), "mempool") != strings.end()); } +BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) +{ + CNetAddr addr; + BOOST_CHECK(LookupHost(std::string("127.0.0.1", 9), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0", 10), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com", 21), addr, false)); + BOOST_CHECK(!LookupHost(std::string("127.0.0.1\0example.com\0", 22), addr, false)); + CSubNet ret; + BOOST_CHECK(LookupSubNet(std::string("1.2.3.0/24", 10), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); + BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); + BOOST_CHECK(LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 2c89a18331..9f928827cb 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 2c56bbdbb0..bcc4a46873 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -26,10 +26,9 @@ extern UniValue read_json(const std::string& jsondata); // Old script.cpp SignatureHash function uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, unsigned int nIn, int nHashType) { - static const uint256 one(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); if (nIn >= txTo.vin.size()) { - return one; + return UINT256_ONE(); } CMutableTransaction txTmp(txTo); @@ -59,7 +58,7 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un unsigned int nOut = nIn; if (nOut >= txTmp.vout.size()) { - return one; + return UINT256_ONE(); } txTmp.vout.resize(nOut+1); for (unsigned int i = 0; i < nOut; i++) diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 188e9986ff..5c6c2ee38e 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,7 +38,7 @@ BOOST_AUTO_TEST_CASE(potential_deadlock_detected) g_debug_lockorder_abort = false; #endif - CCriticalSection rmutex1, rmutex2; + RecursiveMutex rmutex1, rmutex2; TestPotentialDeadLockDetected(rmutex1, rmutex2); Mutex mutex1, mutex2; diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 0939803953..fb45ce0ee6 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -831,6 +831,7 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) reason.clear(); BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); BOOST_CHECK_EQUAL(reason, "bare-multisig"); + fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 67f45c4ed4..7842594b80 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,9 +13,9 @@ #include <boost/test/unit_test.hpp> -bool CheckInputs(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks); -BOOST_AUTO_TEST_SUITE(tx_validationcache_tests) +BOOST_AUTO_TEST_SUITE(txvalidationcache_tests) BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) { @@ -95,8 +95,8 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), 0U); } -// Run CheckInputs (using CoinsTip()) on the given transaction, for all script -// flags. Test that CheckInputs passes for all flags that don't overlap with +// Run CheckInputScripts (using CoinsTip()) on the given transaction, for all script +// flags. Test that CheckInputScripts passes for all flags that don't overlap with // the failing_flags argument, but otherwise fails. // CHECKLOCKTIMEVERIFY and CHECKSEQUENCEVERIFY (and future NOP codes that may // get reassigned) have an interaction with DISCOURAGE_UPGRADABLE_NOPS: if @@ -123,8 +123,8 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail // WITNESS requires P2SH test_flags |= SCRIPT_VERIFY_P2SH; } - bool ret = CheckInputs(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, nullptr); - // CheckInputs should succeed iff test_flags doesn't intersect with + bool ret = CheckInputScripts(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, nullptr); + // CheckInputScripts should succeed iff test_flags doesn't intersect with // failing_flags bool expected_return_value = !(test_flags & failing_flags); BOOST_CHECK_EQUAL(ret, expected_return_value); @@ -133,13 +133,13 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail if (ret && add_to_cache) { // Check that we get a cache hit if the tx was valid std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK(scriptchecks.empty()); } else { // Check that we get script executions to check, if the transaction // was invalid, or we didn't add to cache. std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(tx, state, &::ChainstateActive().CoinsTip(), test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), tx.vin.size()); } } @@ -147,7 +147,7 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) { - // Test that passing CheckInputs with one set of script flags doesn't imply + // Test that passing CheckInputScripts with one set of script flags doesn't imply // that we would pass again with a different set of flags. { LOCK(cs_main); @@ -202,16 +202,16 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) TxValidationState state; PrecomputedTransactionData ptd_spend_tx(spend_tx); - BOOST_CHECK(!CheckInputs(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); + BOOST_CHECK(!CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); // If we call again asking for scriptchecks (as happens in // ConnectBlock), we should add a script check object for this -- we're // not caching invalidity (if that changes, delete this test case). std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), 1U); - // Test that CheckInputs returns true iff DERSIG-enforcing flags are + // Test that CheckInputScripts returns true iff DERSIG-enforcing flags are // not present. Don't add these checks to the cache, so that we can // test later that block validation works fine in the absence of cached // successes. @@ -270,7 +270,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; PrecomputedTransactionData txdata(invalid_with_cltv_tx); - BOOST_CHECK(CheckInputs(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } // TEST CHECKSEQUENCEVERIFY @@ -298,12 +298,12 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; PrecomputedTransactionData txdata(invalid_with_csv_tx); - BOOST_CHECK(CheckInputs(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } // TODO: add tests for remaining script flags - // Test that passing CheckInputs with a valid witness doesn't imply success + // Test that passing CheckInputScripts with a valid witness doesn't imply success // for the same tx with a different witness. { CMutableTransaction valid_with_witness_tx; @@ -360,12 +360,12 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) TxValidationState state; PrecomputedTransactionData txdata(tx); // This transaction is now invalid under segwit, because of the second input. - BOOST_CHECK(!CheckInputs(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); + BOOST_CHECK(!CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); std::vector<CScriptCheck> scriptchecks; // Make sure this transaction was not cached (ie because the first // input was valid) - BOOST_CHECK(CheckInputs(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); // Should get 2 script checks back -- caching is on a whole-transaction basis. BOOST_CHECK_EQUAL(scriptchecks.size(), 2U); } diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 30f0f5d7e6..1df6844062 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -8,22 +8,23 @@ #include <consensus/merkle.h> #include <key_io.h> #include <miner.h> +#include <node/context.h> #include <pow.h> #include <script/standard.h> #include <validation.h> -CTxIn generatetoaddress(const std::string& address) +CTxIn generatetoaddress(const NodeContext& node, const std::string& address) { const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); const auto coinbase_script = GetScriptForDestination(dest); - return MineBlock(coinbase_script); + return MineBlock(node, coinbase_script); } -CTxIn MineBlock(const CScript& coinbase_scriptPubKey) +CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { - auto block = PrepareBlock(coinbase_scriptPubKey); + auto block = PrepareBlock(node, coinbase_scriptPubKey); while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { ++block->nNonce; @@ -36,10 +37,11 @@ CTxIn MineBlock(const CScript& coinbase_scriptPubKey) return CTxIn{block->vtx[0]->GetHash(), 0}; } -std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey) +std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { + assert(node.mempool); auto block = std::make_shared<CBlock>( - BlockAssembler{Params()} + BlockAssembler{*node.mempool, Params()} .CreateNewBlock(coinbase_scriptPubKey) ->block); diff --git a/src/test/util/mining.h b/src/test/util/mining.h index afe4de684f..5f250fffe8 100644 --- a/src/test/util/mining.h +++ b/src/test/util/mining.h @@ -11,14 +11,15 @@ class CBlock; class CScript; class CTxIn; +struct NodeContext; /** Returns the generated coin */ -CTxIn MineBlock(const CScript& coinbase_scriptPubKey); +CTxIn MineBlock(const NodeContext&, const CScript& coinbase_scriptPubKey); /** Prepare a block to be mined */ -std::shared_ptr<CBlock> PrepareBlock(const CScript& coinbase_scriptPubKey); +std::shared_ptr<CBlock> PrepareBlock(const NodeContext&, const CScript& coinbase_scriptPubKey); /** RPC-like helper function, returns the generated coin */ -CTxIn generatetoaddress(const std::string& address); +CTxIn generatetoaddress(const NodeContext&, const std::string& address); #endif // BITCOIN_TEST_UTIL_MINING_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 86c355fdcd..ccb3064d59 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -71,6 +71,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName) SelectParams(chainName); SeedInsecureRand(); gArgs.ForceSetArg("-printtoconsole", "0"); + if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(); LogInstance().StartLogging(); SHA256AutoDetect(); @@ -175,7 +176,7 @@ TestChain100Setup::TestChain100Setup() CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); CBlock& block = pblocktemplate->block; // Replace mempool-selected txns with just coinbase plus passed-in txns: diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 1e2e059a56..6741be8480 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -18,6 +18,9 @@ #include <boost/thread.hpp> +/** This is connected to the logger. Can be used to redirect logs to any other log */ +extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; + // Enable BOOST_CHECK_EQUAL for enum class types template <typename T> std::ostream& operator<<(typename std::enable_if<std::is_enum<T>::value, std::ostream>::type& stream, const T& e) diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 226d2df6e4..fd6012e9fe 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -27,8 +27,7 @@ std::string getnewaddress(CWallet& w) void importaddress(CWallet& wallet, const std::string& address) { auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK(wallet.cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); const auto script = GetScriptForDestination(dest); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 7b024ba75f..42c2c50fa5 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,7 +38,7 @@ BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_criticalsection) { - CCriticalSection cs; + RecursiveMutex cs; do { LOCK(cs); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index 33f840d9c5..78dbf848bb 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 1fa48b325c..dae389a167 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -20,7 +20,17 @@ static const std::vector<unsigned char> V_OP_TRUE{OP_TRUE}; -BOOST_FIXTURE_TEST_SUITE(validation_block_tests, RegTestingSetup) +namespace validation_block_tests { +struct MinerTestingSetup : public RegTestingSetup { + std::shared_ptr<CBlock> Block(const uint256& prev_hash); + std::shared_ptr<const CBlock> GoodBlock(const uint256& prev_hash); + std::shared_ptr<const CBlock> BadBlock(const uint256& prev_hash); + std::shared_ptr<CBlock> FinalizeBlock(std::shared_ptr<CBlock> pblock); + void BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks); +}; +} // namespace validation_block_tests + +BOOST_FIXTURE_TEST_SUITE(validation_block_tests, MinerTestingSetup) struct TestSubscriber : public CValidationInterface { uint256 m_expected_tip; @@ -49,7 +59,7 @@ struct TestSubscriber : public CValidationInterface { } }; -std::shared_ptr<CBlock> Block(const uint256& prev_hash) +std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) { static int i = 0; static uint64_t time = Params().GenesisBlock().nTime; @@ -57,7 +67,7 @@ std::shared_ptr<CBlock> Block(const uint256& prev_hash) CScript pubKey; pubKey << i++ << OP_TRUE; - auto ptemplate = BlockAssembler(Params()).CreateNewBlock(pubKey); + auto ptemplate = BlockAssembler(*m_node.mempool, Params()).CreateNewBlock(pubKey); auto pblock = std::make_shared<CBlock>(ptemplate->block); pblock->hashPrevBlock = prev_hash; pblock->nTime = ++time; @@ -83,7 +93,7 @@ std::shared_ptr<CBlock> Block(const uint256& prev_hash) return pblock; } -std::shared_ptr<CBlock> FinalizeBlock(std::shared_ptr<CBlock> pblock) +std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> pblock) { LOCK(cs_main); // For LookupBlockIndex GenerateCoinbaseCommitment(*pblock, LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); @@ -98,13 +108,13 @@ std::shared_ptr<CBlock> FinalizeBlock(std::shared_ptr<CBlock> pblock) } // construct a valid block -std::shared_ptr<const CBlock> GoodBlock(const uint256& prev_hash) +std::shared_ptr<const CBlock> MinerTestingSetup::GoodBlock(const uint256& prev_hash) { return FinalizeBlock(Block(prev_hash)); } // construct an invalid block (but with a valid header) -std::shared_ptr<const CBlock> BadBlock(const uint256& prev_hash) +std::shared_ptr<const CBlock> MinerTestingSetup::BadBlock(const uint256& prev_hash) { auto pblock = Block(prev_hash); @@ -119,7 +129,7 @@ std::shared_ptr<const CBlock> BadBlock(const uint256& prev_hash) return ret; } -void BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks) +void MinerTestingSetup::BuildChain(const uint256& root, int height, const unsigned int invalid_rate, const unsigned int branch_rate, const unsigned int max_size, std::vector<std::shared_ptr<const CBlock>>& blocks) { if (height <= 0 || blocks.size() >= max_size) return; diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp new file mode 100644 index 0000000000..ab8b957f7d --- /dev/null +++ b/src/test/validation_flush_tests.cpp @@ -0,0 +1,174 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <txmempool.h> +#include <validation.h> +#include <sync.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) + +//! Test utilities for detecting when we need to flush the coins cache based +//! on estimated memory usage. +//! +//! @sa CChainState::GetCoinsCacheSizeState() +//! +BOOST_AUTO_TEST_CASE(getcoinscachesizestate) +{ + BlockManager blockman{}; + CChainState chainstate{blockman}; + chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); + WITH_LOCK(::cs_main, chainstate.InitCoinsCache()); + CTxMemPool tx_pool{}; + + constexpr bool is_64_bit = sizeof(void*) == 8; + + LOCK(::cs_main); + auto& view = chainstate.CoinsTip(); + + //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. + auto add_coin = [](CCoinsViewCache& coins_view) -> COutPoint { + Coin newcoin; + uint256 txid = InsecureRand256(); + COutPoint outp{txid, 0}; + newcoin.nHeight = 1; + newcoin.out.nValue = InsecureRand32(); + newcoin.out.scriptPubKey.assign((uint32_t)56, 1); + coins_view.AddCoin(outp, std::move(newcoin), false); + + return outp; + }; + + // The number of bytes consumed by coin's heap data, i.e. CScript + // (prevector<28, unsigned char>) when assigned 56 bytes of data per above. + // + // See also: Coin::DynamicMemoryUsage(). + constexpr int COIN_SIZE = is_64_bit ? 80 : 64; + + auto print_view_mem_usage = [](CCoinsViewCache& view) { + BOOST_TEST_MESSAGE("CCoinsViewCache memory usage: " << view.DynamicMemoryUsage()); + }; + + constexpr size_t MAX_COINS_CACHE_BYTES = 1024; + + // Without any coins in the cache, we shouldn't need to flush. + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::OK); + + // If the initial memory allocations of cacheCoins don't match these common + // cases, we can't really continue to make assertions about memory usage. + // End the test early. + if (view.DynamicMemoryUsage() != 32 && view.DynamicMemoryUsage() != 16) { + // Add a bunch of coins to see that we at least flip over to CRITICAL. + + for (int i{0}; i < 1000; ++i) { + COutPoint res = add_coin(view); + BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); + } + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::CRITICAL); + + BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); + return; + } + + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(view.DynamicMemoryUsage(), is_64_bit ? 32 : 16); + + // We should be able to add COINS_UNTIL_CRITICAL coins to the cache before going CRITICAL. + // This is contingent not only on the dynamic memory usage of the Coins + // that we're adding (COIN_SIZE bytes per), but also on how much memory the + // cacheCoins (unordered_map) preallocates. + // + // I came up with the count by examining the printed memory usage of the + // CCoinsCacheView, so it's sort of arbitrary - but it shouldn't change + // unless we somehow change the way the cacheCoins map allocates memory. + // + constexpr int COINS_UNTIL_CRITICAL = is_64_bit ? 4 : 5; + + for (int i{0}; i < COINS_UNTIL_CRITICAL; ++i) { + COutPoint res = add_coin(view); + print_view_mem_usage(view); + BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::OK); + } + + // Adding an additional coin will push us over the edge to CRITICAL. + add_coin(view); + print_view_mem_usage(view); + + auto size_state = chainstate.GetCoinsCacheSizeState( + tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0); + + if (!is_64_bit && size_state == CoinsCacheSizeState::LARGE) { + // On 32 bit hosts, we may hit LARGE before CRITICAL. + add_coin(view); + print_view_mem_usage(view); + } + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + CoinsCacheSizeState::CRITICAL); + + // Passing non-zero max mempool usage should allow us more headroom. + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + CoinsCacheSizeState::OK); + + for (int i{0}; i < 3; ++i) { + add_coin(view); + print_view_mem_usage(view); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + CoinsCacheSizeState::OK); + } + + // Adding another coin with the additional mempool room will put us >90% + // but not yet critical. + add_coin(view); + print_view_mem_usage(view); + + // Only perform these checks on 64 bit hosts; I haven't done the math for 32. + if (is_64_bit) { + float usage_percentage = (float)view.DynamicMemoryUsage() / (MAX_COINS_CACHE_BYTES + (1 << 10)); + BOOST_TEST_MESSAGE("CoinsTip usage percentage: " << usage_percentage); + BOOST_CHECK(usage_percentage >= 0.9); + BOOST_CHECK(usage_percentage < 1); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + CoinsCacheSizeState::LARGE); + } + + // Using the default max_* values permits way more coins to be added. + for (int i{0}; i < 1000; ++i) { + add_coin(view); + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool), + CoinsCacheSizeState::OK); + } + + // Flushing the view doesn't take us back to OK because cacheCoins has + // preallocated memory that doesn't get reclaimed even after flush. + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + CoinsCacheSizeState::CRITICAL); + + view.SetBestBlock(InsecureRand256()); + BOOST_CHECK(view.Flush()); + print_view_mem_usage(view); + + BOOST_CHECK_EQUAL( + chainstate.GetCoinsCacheSizeState(tx_pool, MAX_COINS_CACHE_BYTES, 0), + CoinsCacheSizeState::CRITICAL); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h index 0654c2ab1f..2665f8a5be 100644 --- a/src/threadinterrupt.h +++ b/src/threadinterrupt.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/threadsafety.h b/src/threadsafety.h index 47e6b2ea38..bb988dfdfd 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/timedata.cpp b/src/timedata.cpp index 9458b9ae0c..942b3cb919 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,7 +16,7 @@ #include <warnings.h> -static CCriticalSection cs_nTimeOffset; +static RecursiveMutex cs_nTimeOffset; static int64_t nTimeOffset GUARDED_BY(cs_nTimeOffset) = 0; /** diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 3f40785c21..84118b36ef 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -501,7 +501,7 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe } return; } - service = LookupNumeric(std::string(service_id+".onion").c_str(), Params().GetDefaultPort()); + service = LookupNumeric(std::string(service_id+".onion"), Params().GetDefaultPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", GetPrivateKeyFile().string()); diff --git a/src/torcontrol.h b/src/torcontrol.h index e1a1a7937a..474a4d87d9 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txdb.cpp b/src/txdb.cpp index a7eb5f9f67..35bbdab00d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -336,7 +336,7 @@ public: vout.assign(vAvail.size(), CTxOut()); for (unsigned int i = 0; i < vAvail.size(); i++) { if (vAvail[i]) - ::Unserialize(s, CTxOutCompressor(vout[i])); + ::Unserialize(s, Using<TxOutCompression>(vout[i])); } // coinbase height ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); diff --git a/src/txdb.h b/src/txdb.h index 05bf4e4453..488c24f935 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,8 +20,6 @@ class CBlockIndex; class CCoinsViewDBCursor; class uint256; -//! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 10; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! -dbbatchsize default (bytes) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 3c967a04c0..5768219f3a 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,7 +23,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe int64_t _nTime, unsigned int _entryHeight, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp) : tx(_tx), nFee(_nFee), nTxWeight(GetTransactionWeight(*tx)), nUsageSize(RecursiveDynamicUsage(tx)), nTime(_nTime), entryHeight(_entryHeight), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) + spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp), m_epoch(0) { nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); @@ -122,8 +122,6 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes // setMemPoolChildren will be updated, an assumption made in // UpdateForDescendants. for (const uint256 &hash : reverse_iterate(vHashesToUpdate)) { - // we cache the in-mempool children to avoid duplicate updates - setEntries setChildren; // calculate children from mapNextTx txiter it = mapTx.find(hash); if (it == mapTx.end()) { @@ -132,17 +130,21 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); // First calculate the children, and update setMemPoolChildren to // include them, and update their setMemPoolParents to include this tx. - for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { - const uint256 &childHash = iter->second->GetHash(); - txiter childIter = mapTx.find(childHash); - assert(childIter != mapTx.end()); - // We can skip updating entries we've encountered before or that - // are in the block (which are already accounted for). - if (setChildren.insert(childIter).second && !setAlreadyIncluded.count(childHash)) { - UpdateChild(it, childIter, true); - UpdateParent(childIter, it, true); + // we cache the in-mempool children to avoid duplicate updates + { + const auto epoch = GetFreshEpoch(); + for (; iter != mapNextTx.end() && iter->first->hash == hash; ++iter) { + const uint256 &childHash = iter->second->GetHash(); + txiter childIter = mapTx.find(childHash); + assert(childIter != mapTx.end()); + // We can skip updating entries we've encountered before or that + // are in the block (which are already accounted for). + if (!visited(childIter) && !setAlreadyIncluded.count(childHash)) { + UpdateChild(it, childIter, true); + UpdateParent(childIter, it, true); + } } - } + } // release epoch guard for UpdateForDescendants UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded); } } @@ -325,7 +327,7 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, } CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) - : nTransactionsUpdated(0), minerPolicyEstimator(estimator) + : nTransactionsUpdated(0), minerPolicyEstimator(estimator), m_epoch(0), m_has_epoch_guard(false) { _clear(); //lock free clear @@ -1105,4 +1107,23 @@ void CTxMemPool::SetIsLoaded(bool loaded) m_is_loaded = loaded; } + +CTxMemPool::EpochGuard CTxMemPool::GetFreshEpoch() const +{ + return EpochGuard(*this); +} +CTxMemPool::EpochGuard::EpochGuard(const CTxMemPool& in) : pool(in) +{ + assert(!pool.m_has_epoch_guard); + ++pool.m_epoch; + pool.m_has_epoch_guard = true; +} + +CTxMemPool::EpochGuard::~EpochGuard() +{ + // prevents stale results being used + ++pool.m_epoch; + pool.m_has_epoch_guard = false; +} + SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 4a7640b78a..de11d626b4 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -30,7 +30,7 @@ #include <boost/signals2/signal.hpp> class CBlockIndex; -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -129,6 +129,7 @@ public: int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable uint64_t m_epoch; //!< epoch when last touched, useful for graph algorithms }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. @@ -453,6 +454,8 @@ private: mutable int64_t lastRollingFeeUpdate; mutable bool blockSinceLastRollingFeeBump; mutable double rollingMinimumFeeRate; //!< minimum fee to get into the pool, decreases exponentially + mutable uint64_t m_epoch; + mutable bool m_has_epoch_guard; void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -736,6 +739,55 @@ private: * removal. */ void removeUnchecked(txiter entry, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); +public: + /** EpochGuard: RAII-style guard for using epoch-based graph traversal algorithms. + * When walking ancestors or descendants, we generally want to avoid + * visiting the same transactions twice. Some traversal algorithms use + * std::set (or setEntries) to deduplicate the transaction we visit. + * However, use of std::set is algorithmically undesirable because it both + * adds an asymptotic factor of O(log n) to traverals cost and triggers O(n) + * more dynamic memory allocations. + * In many algorithms we can replace std::set with an internal mempool + * counter to track the time (or, "epoch") that we began a traversal, and + * check + update a per-transaction epoch for each transaction we look at to + * determine if that transaction has not yet been visited during the current + * traversal's epoch. + * Algorithms using std::set can be replaced on a one by one basis. + * Both techniques are not fundamentally incomaptible across the codebase. + * Generally speaking, however, the remaining use of std::set for mempool + * traversal should be viewed as a TODO for replacement with an epoch based + * traversal, rather than a preference for std::set over epochs in that + * algorithm. + */ + class EpochGuard { + const CTxMemPool& pool; + public: + EpochGuard(const CTxMemPool& in); + ~EpochGuard(); + }; + // N.B. GetFreshEpoch modifies mutable state via the EpochGuard construction + // (and later destruction) + EpochGuard GetFreshEpoch() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + /** visited marks a CTxMemPoolEntry as having been traversed + * during the lifetime of the most recently created EpochGuard + * and returns false if we are the first visitor, true otherwise. + * + * An EpochGuard must be held when visited is called or an assert will be + * triggered. + * + */ + bool visited(txiter it) const EXCLUSIVE_LOCKS_REQUIRED(cs) { + assert(m_has_epoch_guard); + bool ret = it->m_epoch >= m_epoch; + it->m_epoch = std::max(it->m_epoch, m_epoch); + return ret; + } + + bool visited(Optional<txiter> it) const EXCLUSIVE_LOCKS_REQUIRED(cs) { + assert(m_has_epoch_guard); + return !it || visited(*it); + } }; /** diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index d310637145..85bb746d19 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2018 The Bitcoin Core developers +// Copyright (c) 2010-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,7 +16,6 @@ struct UISignals { boost::signals2::signal<CClientUIInterface::NotifyNumConnectionsChangedSig> NotifyNumConnectionsChanged; boost::signals2::signal<CClientUIInterface::NotifyNetworkActiveChangedSig> NotifyNetworkActiveChanged; boost::signals2::signal<CClientUIInterface::NotifyAlertChangedSig> NotifyAlertChanged; - boost::signals2::signal<CClientUIInterface::LoadWalletSig> LoadWallet; boost::signals2::signal<CClientUIInterface::ShowProgressSig> ShowProgress; boost::signals2::signal<CClientUIInterface::NotifyBlockTipSig> NotifyBlockTip; boost::signals2::signal<CClientUIInterface::NotifyHeaderTipSig> NotifyHeaderTip; @@ -36,7 +35,6 @@ ADD_SIGNALS_IMPL_WRAPPER(InitMessage); ADD_SIGNALS_IMPL_WRAPPER(NotifyNumConnectionsChanged); ADD_SIGNALS_IMPL_WRAPPER(NotifyNetworkActiveChanged); ADD_SIGNALS_IMPL_WRAPPER(NotifyAlertChanged); -ADD_SIGNALS_IMPL_WRAPPER(LoadWallet); ADD_SIGNALS_IMPL_WRAPPER(ShowProgress); ADD_SIGNALS_IMPL_WRAPPER(NotifyBlockTip); ADD_SIGNALS_IMPL_WRAPPER(NotifyHeaderTip); @@ -48,7 +46,6 @@ void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_s void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } -void CClientUIInterface::LoadWallet(std::unique_ptr<interfaces::Wallet>& wallet) { return g_ui_signals.LoadWallet(wallet); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } diff --git a/src/ui_interface.h b/src/ui_interface.h index 9efc2db391..b402177b85 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,10 +17,6 @@ class connection; } } // namespace boost -namespace interfaces { -class Wallet; -} // namespace interfaces - /** General change type (added, updated, removed). */ enum ChangeType { @@ -105,9 +101,6 @@ public: */ ADD_SIGNALS_DECL_WRAPPER(NotifyAlertChanged, void, ); - /** A wallet has been loaded. */ - ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::unique_ptr<interfaces::Wallet>& wallet); - /** * Show progress e.g. for verifychain. * resume_possible indicates shutting down now will result in the current progress action resuming upon restart. diff --git a/src/uint256.cpp b/src/uint256.cpp index ee597e1877..a943e71062 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -75,3 +75,8 @@ template std::string base_blob<256>::GetHex() const; template std::string base_blob<256>::ToString() const; template void base_blob<256>::SetHex(const char*); template void base_blob<256>::SetHex(const std::string&); + +uint256& UINT256_ONE() { + static uint256* one = new uint256(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); + return *one; +} diff --git a/src/uint256.h b/src/uint256.h index 60c5e0554c..b36598f572 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -144,4 +144,6 @@ inline uint256 uint256S(const std::string& str) return rv; } +uint256& UINT256_ONE(); + #endif // BITCOIN_UINT256_H diff --git a/src/undo.h b/src/undo.h index 3f50f4caad..2009c721ab 100644 --- a/src/undo.h +++ b/src/undo.h @@ -32,7 +32,7 @@ public: // Required to maintain compatibility with older undo format. ::Serialize(s, (unsigned char)0); } - ::Serialize(s, CTxOutCompressor(REF(txout->out))); + ::Serialize(s, Using<TxOutCompression>(REF(txout->out))); } explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {} @@ -56,7 +56,7 @@ public: unsigned int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, CTxOutCompressor(REF(txout->out))); + ::Unserialize(s, Using<TxOutCompression>(REF(txout->out))); } explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {} diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp new file mode 100644 index 0000000000..60bd27bf90 --- /dev/null +++ b/src/util/asmap.cpp @@ -0,0 +1,103 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <vector> +#include <assert.h> +#include <crypto/common.h> + +namespace { + +uint32_t DecodeBits(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos, uint8_t minval, const std::vector<uint8_t> &bit_sizes) +{ + uint32_t val = minval; + bool bit; + for (std::vector<uint8_t>::const_iterator bit_sizes_it = bit_sizes.begin(); + bit_sizes_it != bit_sizes.end(); ++bit_sizes_it) { + if (bit_sizes_it + 1 != bit_sizes.end()) { + if (bitpos == endpos) break; + bit = *bitpos; + bitpos++; + } else { + bit = 0; + } + if (bit) { + val += (1 << *bit_sizes_it); + } else { + for (int b = 0; b < *bit_sizes_it; b++) { + if (bitpos == endpos) break; + bit = *bitpos; + bitpos++; + val += bit << (*bit_sizes_it - 1 - b); + } + return val; + } + } + return -1; +} + +const std::vector<uint8_t> TYPE_BIT_SIZES{0, 0, 1}; +uint32_t DecodeType(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos) +{ + return DecodeBits(bitpos, endpos, 0, TYPE_BIT_SIZES); +} + +const std::vector<uint8_t> ASN_BIT_SIZES{15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; +uint32_t DecodeASN(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos) +{ + return DecodeBits(bitpos, endpos, 1, ASN_BIT_SIZES); +} + + +const std::vector<uint8_t> MATCH_BIT_SIZES{1, 2, 3, 4, 5, 6, 7, 8}; +uint32_t DecodeMatch(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos) +{ + return DecodeBits(bitpos, endpos, 2, MATCH_BIT_SIZES); +} + + +const std::vector<uint8_t> JUMP_BIT_SIZES{5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; +uint32_t DecodeJump(std::vector<bool>::const_iterator& bitpos, const std::vector<bool>::const_iterator& endpos) +{ + return DecodeBits(bitpos, endpos, 17, JUMP_BIT_SIZES); +} + +} + +uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip) +{ + std::vector<bool>::const_iterator pos = asmap.begin(); + const std::vector<bool>::const_iterator endpos = asmap.end(); + uint8_t bits = ip.size(); + uint32_t default_asn = 0; + uint32_t opcode, jump, match, matchlen; + while (pos != endpos) { + opcode = DecodeType(pos, endpos); + if (opcode == 0) { + return DecodeASN(pos, endpos); + } else if (opcode == 1) { + jump = DecodeJump(pos, endpos); + if (bits == 0) break; + if (ip[ip.size() - bits]) { + if (jump >= endpos - pos) break; + pos += jump; + } + bits--; + } else if (opcode == 2) { + match = DecodeMatch(pos, endpos); + matchlen = CountBits(match) - 1; + for (uint32_t bit = 0; bit < matchlen; bit++) { + if (bits == 0) break; + if ((ip[ip.size() - bits]) != ((match >> (matchlen - 1 - bit)) & 1)) { + return default_asn; + } + bits--; + } + } else if (opcode == 3) { + default_asn = DecodeASN(pos, endpos); + } else { + break; + } + } + return 0; // 0 is not a valid ASN +} diff --git a/src/util/asmap.h b/src/util/asmap.h new file mode 100644 index 0000000000..a0e14013c5 --- /dev/null +++ b/src/util/asmap.h @@ -0,0 +1,10 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ASMAP_H +#define BITCOIN_UTIL_ASMAP_H + +uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip); + +#endif // BITCOIN_UTIL_ASMAP_H diff --git a/src/util/error.cpp b/src/util/error.cpp index aa44ed3e3a..72a6e87cde 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2018 The Bitcoin Core developers +// Copyright (c) 2010-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/error.h b/src/util/error.h index f540b0020d..61af88ddea 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -1,4 +1,4 @@ -// Copyright (c) 2010-2018 The Bitcoin Core developers +// Copyright (c) 2010-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/fees.cpp b/src/util/fees.cpp index 41149888d7..b335bfa666 100644 --- a/src/util/fees.cpp +++ b/src/util/fees.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/fees.h b/src/util/fees.h index fc355ce9c2..a930c8935a 100644 --- a/src/util/fees.h +++ b/src/util/fees.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_UTIL_FEES_H diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 3e75a2e3e9..2797f450ca 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/moneystr.h b/src/util/moneystr.h index 4d0218911a..027c7e2e53 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/rbf.cpp b/src/util/rbf.cpp index d520a9606d..ef536bdcad 100644 --- a/src/util/rbf.cpp +++ b/src/util/rbf.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/rbf.h b/src/util/rbf.h index d3ef110628..6a20b37de5 100644 --- a/src/util/rbf.h +++ b/src/util/rbf.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/settings.h b/src/util/settings.h index 9ca581109d..1d03639fa2 100644 --- a/src/util/settings.h +++ b/src/util/settings.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -71,11 +71,11 @@ struct SettingsSpan { explicit SettingsSpan(const SettingsValue& value) noexcept : SettingsSpan(&value, 1) {} explicit SettingsSpan(const SettingsValue* data, size_t size) noexcept : data(data), size(size) {} explicit SettingsSpan(const std::vector<SettingsValue>& vec) noexcept; - const SettingsValue* begin() const; //<! Pointer to first non-negated value. - const SettingsValue* end() const; //<! Pointer to end of values. - bool empty() const; //<! True if there are any non-negated values. - bool last_negated() const; //<! True if the last value is negated. - size_t negated() const; //<! Number of negated values. + const SettingsValue* begin() const; //!< Pointer to first non-negated value. + const SettingsValue* end() const; //!< Pointer to end of values. + bool empty() const; //!< True if there are any non-negated values. + bool last_negated() const; //!< True if the last value is negated. + size_t negated() const; //!< Number of negated values. const SettingsValue* data = nullptr; size_t size = 0; diff --git a/src/util/spanparsing.cpp b/src/util/spanparsing.cpp index 0c8575399a..0f68254f2c 100644 --- a/src/util/spanparsing.cpp +++ b/src/util/spanparsing.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/spanparsing.h b/src/util/spanparsing.h index 63f54758bd..fa2e698e6d 100644 --- a/src/util/spanparsing.h +++ b/src/util/spanparsing.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 31719cd975..eec1a52e95 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.h b/src/util/strencodings.h index e35b2ab857..ccc4edac12 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/system.cpp b/src/util/system.cpp index 5587764c58..13ff99c663 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -64,6 +64,7 @@ #endif #include <thread> +#include <typeinfo> #include <univalue.h> // Application startup time (used for uptime calculation) @@ -860,6 +861,32 @@ std::vector<util::SettingsValue> ArgsManager::GetSettingsList(const std::string& return util::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); } +void ArgsManager::logArgsPrefix( + const std::string& prefix, + const std::string& section, + const std::map<std::string, std::vector<util::SettingsValue>>& args) const +{ + std::string section_str = section.empty() ? "" : "[" + section + "] "; + for (const auto& arg : args) { + for (const auto& value : arg.second) { + Optional<unsigned int> flags = GetArgFlags('-' + arg.first); + if (flags) { + std::string value_str = (*flags & SENSITIVE) ? "****" : value.write(); + LogPrintf("%s %s%s=%s\n", prefix, section_str, arg.first, value_str); + } + } + } +} + +void ArgsManager::LogArgs() const +{ + LOCK(cs_args); + for (const auto& section : m_settings.ro_config) { + logArgsPrefix("Config file arg:", section.first, section.second); + } + logArgsPrefix("Command-line arg:", "", m_settings.command_line_options); +} + bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 @@ -970,17 +997,19 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { SetEndOfFile(hFile); #elif defined(MAC_OSX) // OSX specific version + // NOTE: Contrary to other OS versions, the OSX version assumes that + // NOTE: offset is the size of the file. fstore_t fst; fst.fst_flags = F_ALLOCATECONTIG; fst.fst_posmode = F_PEOFPOSMODE; fst.fst_offset = 0; - fst.fst_length = (off_t)offset + length; + fst.fst_length = length; // mac os fst_length takes the # of free bytes to allocate, not desired file size fst.fst_bytesalloc = 0; if (fcntl(fileno(file), F_PREALLOCATE, &fst) == -1) { fst.fst_flags = F_ALLOCATEALL; fcntl(fileno(file), F_PREALLOCATE, &fst); } - ftruncate(fileno(file), fst.fst_length); + ftruncate(fileno(file), static_cast<off_t>(offset) + length); #else #if defined(__linux__) // Version using posix_fallocate diff --git a/src/util/system.h b/src/util/system.h index 394adb9555..bb69181de9 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -145,6 +145,8 @@ public: * between mainnet and regtest/testnet won't cause problems due to these * parameters by accident. */ NETWORK_ONLY = 0x200, + // This argument's value is sensitive (such as a password). + SENSITIVE = 0x400, }; protected: @@ -155,7 +157,7 @@ protected: unsigned int m_flags; }; - mutable CCriticalSection cs_args; + mutable RecursiveMutex cs_args; util::Settings m_settings GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); @@ -318,6 +320,19 @@ public: * Return nullopt for unknown arg. */ Optional<unsigned int> GetArgFlags(const std::string& name) const; + + /** + * Log the config file options and the command line arguments, + * useful for troubleshooting. + */ + void LogArgs() const; + +private: + // Helper function for LogArgs(). + void logArgsPrefix( + const std::string& prefix, + const std::string& section, + const std::map<std::string, std::vector<util::SettingsValue>>& args) const; }; extern ArgsManager gArgs; diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index 20df403a66..764fffabd7 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/threadnames.h b/src/util/threadnames.h index 69a1b55bfe..64b2689cf1 100644 --- a/src/util/threadnames.h +++ b/src/util/threadnames.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/url.cpp b/src/util/url.cpp index 49eacbf2d0..e42d93bce8 100644 --- a/src/util/url.cpp +++ b/src/util/url.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/url.h b/src/util/url.h index 3d7315a338..e9ea2ab765 100644 --- a/src/util/url.h +++ b/src/util/url.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/validation.cpp b/src/util/validation.cpp index bd52f57751..89bc6665a4 100644 --- a/src/util/validation.cpp +++ b/src/util/validation.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,12 +8,18 @@ #include <consensus/validation.h> #include <tinyformat.h> -/** Convert ValidationState to a human-readable message for logging */ std::string FormatStateMessage(const ValidationState &state) { - return strprintf("%s%s", - state.GetRejectReason(), - state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage()); + if (state.IsValid()) { + return "Valid"; + } + + const std::string debug_message = state.GetDebugMessage(); + if (!debug_message.empty()) { + return strprintf("%s, %s", state.GetRejectReason(), debug_message); + } + + return state.GetRejectReason(); } const std::string strMessageMagic = "Bitcoin Signed Message:\n"; diff --git a/src/validation.cpp b/src/validation.cpp index 8f5d333170..bab04b8e34 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -133,7 +133,7 @@ CTxMemPool mempool(&feeEstimator); namespace { CBlockIndex* pindexBestInvalid = nullptr; - CCriticalSection cs_LastBlockFile; + RecursiveMutex cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; int nLastBlockFile = 0; /** Global flag to indicate we should check to see if there are @@ -180,7 +180,7 @@ std::unique_ptr<CBlockTreeDB> pblocktree; // See definition for documentation static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); -bool CheckInputs(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); static FlatFileSeq BlockFileSeq(); static FlatFileSeq UndoFileSeq(); @@ -396,19 +396,19 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationS // pool.cs should be locked already, but go ahead and re-take the lock here // to enforce that mempool doesn't change between when we check the view - // and when we actually call through to CheckInputs + // and when we actually call through to CheckInputScripts LOCK(pool.cs); assert(!tx.IsCoinBase()); for (const CTxIn& txin : tx.vin) { const Coin& coin = view.AccessCoin(txin.prevout); - // At this point we haven't actually checked if the coins are all - // available (or shouldn't assume we have, since CheckInputs does). - // So we just return failure if the inputs are not available here, - // and then only have to check equivalence for available inputs. + // AcceptToMemoryPoolWorker has already checked that the coins are + // available, so this shouldn't fail. If the inputs are not available + // here then return false. if (coin.IsSpent()) return false; + // Check equivalence for available inputs. const CTransactionRef& txFrom = pool.get(txin.prevout.hash); if (txFrom) { assert(txFrom->GetHash() == txin.prevout.hash); @@ -421,8 +421,8 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationS } } - // Call CheckInputs() to cache signature and script validity against current tip consensus rules. - return CheckInputs(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); + // Call CheckInputScripts() to cache signature and script validity against current tip consensus rules. + return CheckInputScripts(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); } namespace { @@ -906,20 +906,20 @@ bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, Precompute constexpr unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; - // Check against previous transactions + // Check input scripts and signatures. // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputs(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { + if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we // need to turn both off, and compare against just turning off CLEANSTACK // to see if the failure is specifically due to witness validation. - TxValidationState state_dummy; // Want reported failures to be from first CheckInputs - if (!tx.HasWitness() && CheckInputs(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && - !CheckInputs(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { + TxValidationState state_dummy; // Want reported failures to be from first CheckInputScripts + if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && + !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { // Only the witness is missing, so the transaction itself may be fine. state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, state.GetRejectReason(), state.GetDebugMessage()); } - return false; // state filled in by CheckInputs + return false; // state filled in by CheckInputScripts } return true; @@ -950,7 +950,7 @@ bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, Precomp // transactions into the mempool can be exploited as a DoS attack. unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(::ChainActive().Tip(), chainparams.GetConsensus()); if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata)) { - return error("%s: BUG! PLEASE REPORT THIS! CheckInputs failed against latest-block but not STANDARD flags %s, %s", + return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } @@ -1392,10 +1392,14 @@ static void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) E CheckForkWarningConditions(); } +// Called both upon regular invalid block discovery *and* InvalidateBlock void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) pindexBestInvalid = pindexNew; + if (pindexBestHeader != nullptr && pindexBestHeader->GetAncestor(pindexNew->nHeight) == pindexNew) { + pindexBestHeader = ::ChainActive().Tip(); + } LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, @@ -1408,6 +1412,8 @@ void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(c CheckForkWarningConditions(); } +// Same as InvalidChainFound, above, except not called directly from InvalidateBlock, +// which does its own setBlockIndexCandidates manageent. void CChainState::InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) { if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; @@ -1466,8 +1472,10 @@ void InitScriptExecutionCache() { } /** - * Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts) - * This does not modify the UTXO set. + * Check whether all of this transaction's input scripts succeed. + * + * This involves ECDSA signature checks so can be computationally intensive. This function should + * only be called after the cheap sanity checks in CheckTxInputs passed. * * If pvChecks is not nullptr, script checks are pushed onto it instead of being performed inline. Any * script checks which are not necessary (eg due to script execution cache hits) are, obviously, @@ -1482,7 +1490,7 @@ void InitScriptExecutionCache() { * * Non-static (and re-declared) in src/test/txvalidationcache_tests.cpp */ -bool CheckInputs(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (tx.IsCoinBase()) return true; @@ -2129,11 +2137,11 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ TxValidationState tx_state; - if (fScriptChecks && !CheckInputs(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); - return error("ConnectBlock(): CheckInputs on %s failed with %s", + return error("ConnectBlock(): CheckInputScripts on %s failed with %s", tx.GetHash().ToString(), FormatStateMessage(state)); } control.Add(vChecks); @@ -2185,13 +2193,44 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return true; } +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_pool) +{ + return this->GetCoinsCacheSizeState( + tx_pool, + nCoinCacheUsage, + gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); +} + +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( + const CTxMemPool& tx_pool, + size_t max_coins_cache_size_bytes, + size_t max_mempool_size_bytes) +{ + int64_t nMempoolUsage = tx_pool.DynamicMemoryUsage(); + int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); + int64_t nTotalSpace = + max_coins_cache_size_bytes + std::max<int64_t>(max_mempool_size_bytes - nMempoolUsage, 0); + + //! No need to periodic flush if at least this much space still available. + static constexpr int64_t MAX_BLOCK_COINSDB_USAGE_BYTES = 10 * 1024 * 1024; // 10MB + int64_t large_threshold = + std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE_BYTES); + + if (cacheSize > nTotalSpace) { + LogPrintf("Cache size (%s) exceeds total space (%s)\n", cacheSize, nTotalSpace); + return CoinsCacheSizeState::CRITICAL; + } else if (cacheSize > large_threshold) { + return CoinsCacheSizeState::LARGE; + } + return CoinsCacheSizeState::OK; +} + bool CChainState::FlushStateToDisk( const CChainParams& chainparams, BlockValidationState &state, FlushStateMode mode, int nManualPruneHeight) { - int64_t nMempoolUsage = mempool.DynamicMemoryUsage(); LOCK(cs_main); assert(this->CanFlushToDisk()); static int64_t nLastWrite = 0; @@ -2206,6 +2245,7 @@ bool CChainState::FlushStateToDisk( { bool fFlushForPrune = false; bool fDoFullFlush = false; + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(::mempool); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { @@ -2234,13 +2274,10 @@ bool CChainState::FlushStateToDisk( if (nLastFlush == 0) { nLastFlush = nNow; } - int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); - int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FlushStateMode::PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); + bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE; // The cache is over the limit, we have to write now. - bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cacheSize > nTotalSpace; + bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. diff --git a/src/validation.h b/src/validation.h index 54f97e7213..a5335edc43 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -136,7 +136,7 @@ struct BlockHasher size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); } }; -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; @@ -530,6 +530,15 @@ public: void InitCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; +enum class CoinsCacheSizeState +{ + //! The coins cache is in immediate need of a flush. + CRITICAL = 2, + //! The cache is at >= 90% capacity. + LARGE = 1, + OK = 0 +}; + /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -551,7 +560,7 @@ private: * Every received block is assigned a unique and increasing identifier, so we * know which one to give priority in case of a fork. */ - CCriticalSection cs_nBlockSequenceId; + RecursiveMutex cs_nBlockSequenceId; /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ int32_t nBlockSequenceId = 1; /** Decreasing counter (used by subsequent preciousblock calls). */ @@ -563,7 +572,7 @@ private: * the ChainState CriticalSection * A lock that must be held when modifying this ChainState - held in ActivateBestChain() */ - CCriticalSection m_cs_chainstate; + RecursiveMutex m_cs_chainstate; /** * Whether this chainstate is undergoing initial block download. @@ -721,6 +730,17 @@ public: /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Dictates whether we need to flush the cache to disk or not. + //! + //! @return the state of the size of the coins cache. + CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool& tx_pool) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + CoinsCacheSizeState GetCoinsCacheSizeState( + const CTxMemPool& tx_pool, + size_t max_coins_cache_size_bytes, + size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + private: bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 6c0f4d5edd..0f513c065f 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -1,12 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <validationinterface.h> +#include <chain.h> +#include <consensus/validation.h> +#include <logging.h> #include <primitives/block.h> +#include <primitives/transaction.h> #include <scheduler.h> +#include <util/validation.h> #include <future> #include <unordered_map> @@ -110,52 +115,89 @@ void SyncWithValidationInterfaceQueue() { promise.get_future().wait(); } +// Use a macro instead of a function for conditional logging to prevent +// evaluating arguments when logging is not enabled. +// +// NOTE: The lambda captures all local variables by value. +#define ENQUEUE_AND_LOG_EVENT(event, fmt, name, ...) \ + do { \ + auto local_name = (name); \ + LOG_EVENT("Enqueuing " fmt, local_name, __VA_ARGS__); \ + m_internals->m_schedulerClient.AddToProcessQueue([=] { \ + LOG_EVENT(fmt, local_name, __VA_ARGS__); \ + event(); \ + }); \ + } while (0) + +#define LOG_EVENT(fmt, ...) \ + LogPrint(BCLog::VALIDATION, fmt "\n", __VA_ARGS__) void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { // Dependencies exist that require UpdatedBlockTip events to be delivered in the order in which // the chain actually updates. One way to ensure this is for the caller to invoke this signal // in the same critical section where the chain is updated - m_internals->m_schedulerClient.AddToProcessQueue([pindexNew, pindexFork, fInitialDownload, this] { + auto event = [pindexNew, pindexFork, fInitialDownload, this] { m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: new block hash=%s fork block hash=%s (in IBD=%s)", __func__, + pindexNew->GetBlockHash().ToString(), + pindexFork ? pindexFork->GetBlockHash().ToString() : "null", + fInitialDownload); } void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + auto event = [ptx, this] { m_internals->TransactionAddedToMempool(ptx); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, + ptx->GetHash().ToString(), + ptx->GetWitnessHash().ToString()); } void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef &ptx) { - m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + auto event = [ptx, this] { m_internals->TransactionRemovedFromMempool(ptx); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, + ptx->GetHash().ToString(), + ptx->GetWitnessHash().ToString()); } void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>>& pvtxConflicted) { - m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, pvtxConflicted, this] { + auto event = [pblock, pindex, pvtxConflicted, this] { m_internals->BlockConnected(pblock, pindex, *pvtxConflicted); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, + pblock->GetHash().ToString(), + pindex->nHeight); } void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { - m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, this] { + auto event = [pblock, pindex, this] { m_internals->BlockDisconnected(pblock, pindex); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s block height=%d", __func__, + pblock->GetHash().ToString(), + pindex->nHeight); } void CMainSignals::ChainStateFlushed(const CBlockLocator &locator) { - m_internals->m_schedulerClient.AddToProcessQueue([locator, this] { + auto event = [locator, this] { m_internals->ChainStateFlushed(locator); - }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block hash=%s", __func__, + locator.IsNull() ? "null" : locator.vHave.front().ToString()); } void CMainSignals::BlockChecked(const CBlock& block, const BlockValidationState& state) { + LOG_EVENT("%s: block hash=%s state=%s", __func__, + block.GetHash().ToString(), FormatStateMessage(state)); m_internals->BlockChecked(block, state); } void CMainSignals::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock> &block) { + LOG_EVENT("%s: block hash=%s", __func__, block->GetHash().ToString()); m_internals->NewPoWValidBlock(pindex, block); } diff --git a/src/validationinterface.h b/src/validationinterface.h index 63aad8661f..ed6c560944 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <functional> #include <memory> -extern CCriticalSection cs_main; +extern RecursiveMutex cs_main; class BlockValidationState; class CBlock; class CBlockIndex; diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 2285579cd9..af07c67ccf 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/versionbits.h b/src/versionbits.h index d8dda7d95b..b02f848b67 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/versionbitsinfo.cpp b/src/versionbitsinfo.cpp index 82df92ac90..20297b9f9d 100644 --- a/src/versionbitsinfo.cpp +++ b/src/versionbitsinfo.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index 14513bc9e9..c83e598825 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 870e235964..5bbb2c0ad0 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 4367a5047f..f59c63260e 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 79825847c8..8b042162d8 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -44,7 +44,7 @@ void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filena } } -CCriticalSection cs_db; +RecursiveMutex cs_db; std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db); //!< Map from directory name to db environment. } // namespace @@ -650,7 +650,7 @@ void BerkeleyEnvironment::ReloadDbEnv() { // Make sure that no Db's are in use AssertLockNotHeld(cs_db); - std::unique_lock<CCriticalSection> lock(cs_db); + std::unique_lock<RecursiveMutex> lock(cs_db); m_db_in_use.wait(lock, [this](){ for (auto& count : mapFileUseCount) { if (count.second > 0) return false; diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 36588eb7d1..b93b9ef1bc 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -47,7 +47,8 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet // check that original tx consists entirely of our inputs // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!wallet.IsAllFromMe(*wtx.tx, ISMINE_SPENDABLE)) { + isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + if (!wallet.IsAllFromMe(*wtx.tx, filter)) { errors.push_back("Transaction contains inputs that don't belong to this wallet"); return feebumper::Result::WALLET_ERROR; } @@ -78,7 +79,8 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt CFeeRate incrementalRelayFee = std::max(wallet.chain().relayIncrementalFee(), CFeeRate(WALLET_INCREMENTAL_RELAY_FEE)); // Given old total fee and transaction size, calculate the old feeRate - CAmount old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + CAmount old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); const int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); CFeeRate nOldFeeRate(old_fee, txSize); // Min total fee is old fee + relay fee @@ -195,7 +197,8 @@ Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, co } // calculate the old fee and fee-rate - old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + isminefilter filter = wallet->GetLegacyScriptPubKeyMan() && wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); CFeeRate nOldFeeRate(old_fee, txSize); // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to // future proof against changes to network wide policy for incremental relay @@ -308,7 +311,8 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } } - old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; + old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); if (coin_control.m_feerate) { // The user provided a feeRate argument. diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index dd0d2ffbd7..2ebc9aba39 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -46,7 +46,7 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data. 0 to entirely disable the fallbackfee feature. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u). Warning: Smaller sizes may increase the risk of losing funds when restoring from an old backup, if none of the addresses in the original keypool have been used.", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 0bc6c90354..5cdd7dff80 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/psbtwallet.cpp b/src/wallet/psbtwallet.cpp index 96c1ad8d3f..d995fb06d4 100644 --- a/src/wallet/psbtwallet.cpp +++ b/src/wallet/psbtwallet.cpp @@ -44,6 +44,9 @@ TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& ps if (!input.witness_utxo.IsNull()) { script = input.witness_utxo.scriptPubKey; } else if (input.non_witness_utxo) { + if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { + return TransactionError::MISSING_INPUTS; + } script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; } else { // There's no UTXO so we can just skip this now @@ -52,21 +55,21 @@ TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& ps } SignatureData sigdata; input.FillSignatureData(sigdata); - const SigningProvider* provider = pwallet->GetSigningProvider(script, sigdata); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(script, sigdata); if (!provider) { complete = false; continue; } - complete &= SignPSBTInput(HidingSigningProvider(provider, !sign, !bip32derivs), psbtx, i, sighash_type); + complete &= SignPSBTInput(HidingSigningProvider(provider.get(), !sign, !bip32derivs), psbtx, i, sighash_type); } // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { const CTxOut& out = psbtx.tx->vout.at(i); - const SigningProvider* provider = pwallet->GetSigningProvider(out.scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(out.scriptPubKey); if (provider) { - UpdatePSBTOutput(HidingSigningProvider(provider, true, !bip32derivs), psbtx, i); + UpdatePSBTOutput(HidingSigningProvider(provider.get(), true, !bip32derivs), psbtx, i); } } diff --git a/src/wallet/psbtwallet.h b/src/wallet/psbtwallet.h index a7e52df6d9..b35a0a58d1 100644 --- a/src/wallet/psbtwallet.h +++ b/src/wallet/psbtwallet.h @@ -15,8 +15,8 @@ * finalize.) Sets `error` and returns false if something goes wrong. * * @param[in] pwallet pointer to a wallet - * @param[in] &psbtx reference to PartiallySignedTransaction to fill in - * @param[out] &complete indicates whether the PSBT is now complete + * @param[in] psbtx PartiallySignedTransaction to fill in + * @param[out] complete indicates whether the PSBT is now complete * @param[in] sighash_type the sighash type to use when signing (if PSBT does not specify) * @param[in] sign whether to sign or not * @param[in] bip32derivs whether to fill in bip32 derivation information if available diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 633ac1b16d..7e704a95fe 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -125,7 +125,7 @@ UniValue importprivkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); WalletRescanReserver reserver(pwallet); bool fRescan = true; @@ -253,7 +253,7 @@ UniValue importaddress(const JSONRPCRequest& request) }, }.Check(request); - EnsureLegacyScriptPubKeyMan(*pwallet); + EnsureLegacyScriptPubKeyMan(*pwallet, true); std::string strLabel; if (!request.params[1].isNull()) @@ -454,7 +454,7 @@ UniValue importpubkey(const JSONRPCRequest& request) }, }.Check(request); - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); std::string strLabel; if (!request.params[1].isNull()) @@ -538,7 +538,7 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.Check(request); - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); if (pwallet->chain().havePruned()) { // Exit early and print an error. @@ -700,7 +700,7 @@ UniValue dumpprivkey(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -751,8 +751,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); - AssertLockHeld(spk_man.cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); EnsureWalletIsUnlocked(pwallet); @@ -1322,8 +1321,19 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) "\"options\""}, }, RPCResult{ - "\nResponse is an array with the same size as the input that has the execution result :\n" - " [{\"success\": true}, {\"success\": true, \"warnings\": [\"Ignoring irrelevant private key\"]}, {\"success\": false, \"error\": {\"code\": -1, \"message\": \"Internal Server Error\"}}, ...]\n" + "[ (json array) Response is an array with the same size as the input that has the execution result\n" + " { (json object)\n" + " \"success\" : true|false, (boolean)\n" + " \"warnings\" : [ (json array, optional)\n" + " \"str\", (string)\n" + " ...\n" + " ],\n" + " \"error\" : { (json object, optional)\n" + " ... JSONRPC error\n" + " },\n" + " },\n" + " ...\n" + "]\n" }, RPCExamples{ HelpExampleCli("importmulti", "'[{ \"scriptPubKey\": { \"address\": \"<my address>\" }, \"timestamp\":1455191478 }, " @@ -1335,7 +1345,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); - EnsureLegacyScriptPubKeyMan(*wallet); + EnsureLegacyScriptPubKeyMan(*wallet, true); const UniValue& requests = mainRequest.params[0]; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 63cbe02b17..00c4a7427a 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -124,9 +124,13 @@ void EnsureWalletIsUnlocked(const CWallet* pwallet) } } -LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet) +// also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create) { LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man && also_create) { + spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + } if (!spk_man) { throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); } @@ -561,7 +565,7 @@ static UniValue signmessage(const JSONRPCRequest& request) } CScript script_pub_key = GetScriptForDestination(*pkhash); - const SigningProvider* provider = pwallet->GetSigningProvider(script_pub_key); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(script_pub_key); if (!provider) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key not available"); } @@ -983,7 +987,7 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); std::string label; if (!request.params[2].isNull()) @@ -1383,7 +1387,7 @@ static const std::string TransactionDescriptionString() " \"time\": xxx, (numeric) The transaction time expressed in " + UNIX_EPOCH_TIME + ".\n" " \"timereceived\": xxx, (numeric) The time received expressed in " + UNIX_EPOCH_TIME + ".\n" " \"comment\": \"...\", (string) If a comment is associated with the transaction, only present if not empty.\n" - " \"bip125-replaceable\": \"yes|no|unknown\", (string) Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" + " \"bip125-replaceable\" : \"str\", (string) (\"yes|no|unknown\") Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" " may be unknown for unconfirmed transactions not in the mempool\n"; } @@ -1531,11 +1535,12 @@ static UniValue listsinceblock(const JSONRPCRequest& request) " (not guaranteed to work on pruned nodes)"}, }, RPCResult{ - "{\n" - " \"transactions\": [\n" + "{ (json object)\n" + " \"transactions\" : [ (json array)\n" + " { (json object)\n" " \"involvesWatchonly\": xxx, (bool) Only returns true if imported addresses were involved in transaction.\n" - " \"address\":\"address\", (string) The bitcoin address of the transaction.\n" - " \"category\": (string) The transaction category.\n" + " \"address\" : \"str\", (string) The bitcoin address of the transaction.\n" + " \"category\" : \"str\", (string) The transaction category.\n" " \"send\" Transactions sent.\n" " \"receive\" Non-coinbase transactions received.\n" " \"generate\" Coinbase transactions received with more than 100 confirmations.\n" @@ -1547,15 +1552,16 @@ static UniValue listsinceblock(const JSONRPCRequest& request) " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the 'send' category of transactions.\n" + TransactionDescriptionString() + " \"abandoned\": xxx, (bool) 'true' if the transaction has been abandoned (inputs are respendable). Only available for the 'send' category of transactions.\n" - " \"comment\": \"...\", (string) If a comment is associated with the transaction.\n" " \"label\" : \"label\" (string) A comment for the address/transaction, if any\n" " \"to\": \"...\", (string) If a comment to is associated with the transaction.\n" + " },\n" + " ...\n" " ],\n" - " \"removed\": [\n" + " \"removed\": [ (json array)\n" " <structure is the same as \"transactions\" above, only present if include_removed=true>\n" " Note: transactions that were re-added in the active chain will appear as-is in this array, and may thus have a positive confirmation count.\n" " ],\n" - " \"lastblock\": \"lastblockhash\" (string) The hash of the block (target_confirmations-1) from the best block on the main chain. This is typically used to feed back into listsinceblock the next time you call it. So you would generally use a target_confirmations of say 6, so you will be continually re-notified of transactions until they've reached 6 confirmations plus any new ones\n" + " \"lastblock\": \"hex\" (string) The hash of the block (target_confirmations-1) from the best block on the main chain. This is typically used to feed back into listsinceblock the next time you call it. So you would generally use a target_confirmations of say 6, so you will be continually re-notified of transactions until they've reached 6 confirmations plus any new ones\n" "}\n" }, RPCExamples{ @@ -2927,7 +2933,7 @@ static UniValue listunspent(const JSONRPCRequest& request) CTxDestination address; const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); - bool reused = avoid_reuse && pwallet->IsUsedDestination(address); + bool reused = avoid_reuse && pwallet->IsUsedDestination(out.tx->GetHash(), out.i); if (destinations.size() && (!fValidAddress || !destinations.count(address))) continue; @@ -2944,7 +2950,7 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("label", i->second.name); } - const SigningProvider* provider = pwallet->GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); if (provider) { if (scriptPubKey.IsPayToScriptHash()) { const CScriptID& hash = CScriptID(boost::get<ScriptHash>(address)); @@ -2984,7 +2990,7 @@ static UniValue listunspent(const JSONRPCRequest& request) entry.pushKV("spendable", out.fSpendable); entry.pushKV("solvable", out.fSolvable); if (out.fSolvable) { - const SigningProvider* provider = pwallet->GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); if (provider) { auto descriptor = InferDescriptor(scriptPubKey, *provider); entry.pushKV("desc", descriptor->ToString()); @@ -3297,21 +3303,21 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) // Parse the prevtxs array ParsePrevouts(request.params[1], nullptr, coins); - std::set<const SigningProvider*> providers; + std::set<std::shared_ptr<SigningProvider>> providers; for (const std::pair<COutPoint, Coin> coin_pair : coins) { - const SigningProvider* provider = pwallet->GetSigningProvider(coin_pair.second.out.scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(coin_pair.second.out.scriptPubKey); if (provider) { providers.insert(std::move(provider)); } } if (providers.size() == 0) { - // When there are no available providers, use DUMMY_SIGNING_PROVIDER so we can check if the tx is complete - providers.insert(&DUMMY_SIGNING_PROVIDER); + // When there are no available providers, use a dummy SigningProvider so we can check if the tx is complete + providers.insert(std::make_shared<SigningProvider>()); } UniValue result(UniValue::VOBJ); - for (const SigningProvider* provider : providers) { - SignTransaction(mtx, provider, coins, request.params[2], result); + for (std::shared_ptr<SigningProvider> provider : providers) { + SignTransaction(mtx, provider.get(), coins, request.params[2], result); } return result; } @@ -3365,10 +3371,11 @@ static UniValue bumpfee(const JSONRPCRequest& request) }, RPCResult{ "{\n" - " \"txid\": \"value\", (string) The id of the new transaction\n" - " \"origfee\": n, (numeric) Fee of the replaced transaction\n" - " \"fee\": n, (numeric) Fee of the new transaction\n" - " \"errors\": [ str... ] (json array of strings) Errors encountered during processing (may be empty)\n" + " \"psbt\": \"psbt\", (string) The base64-encoded unsigned PSBT of the new transaction. Only returned when wallet private keys are disabled.\n" + " \"txid\": \"value\", (string) The id of the new transaction. Only returned when wallet private keys are enabled.\n" + " \"origfee\": n, (numeric) The fee of the replaced transaction.\n" + " \"fee\": n, (numeric) The fee of the new transaction.\n" + " \"errors\": [ str... ] (json array of strings) Errors encountered during processing (may be empty).\n" "}\n" }, RPCExamples{ @@ -3380,10 +3387,12 @@ static UniValue bumpfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); uint256 hash(ParseHashV(request.params[0], "txid")); + CCoinControl coin_control; + coin_control.fAllowWatchOnly = pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); // optional parameters CAmount totalFee = 0; - CCoinControl coin_control; coin_control.m_signal_bip125_rbf = true; + if (!request.params[1].isNull()) { UniValue options = request.params[1]; RPCTypeCheckObj(options, @@ -3468,17 +3477,32 @@ static UniValue bumpfee(const JSONRPCRequest& request) } } - // sign bumped transaction - if (!feebumper::SignTransaction(*pwallet, mtx)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); - } - // commit the bumped transaction - uint256 txid; - if (feebumper::CommitTransaction(*pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { - throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); - } UniValue result(UniValue::VOBJ); - result.pushKV("txid", txid.GetHex()); + + // If wallet private keys are enabled, return the new transaction id, + // otherwise return the base64-encoded unsigned PSBT of the new transaction. + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!feebumper::SignTransaction(*pwallet, mtx)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); + } + + uint256 txid; + if (feebumper::CommitTransaction(*pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { + throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); + } + + result.pushKV("txid", txid.GetHex()); + } else { + PartiallySignedTransaction psbtx(mtx); + bool complete = false; + const TransactionError err = FillPSBT(pwallet, psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + CHECK_NONFATAL(err == TransactionError::OK); + CHECK_NONFATAL(!complete); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + result.pushKV("psbt", EncodeBase64(ssTx.str())); + } + result.pushKV("origfee", ValueFromAmount(old_fee)); result.pushKV("fee", ValueFromAmount(new_fee)); UniValue result_errors(UniValue::VARR); @@ -3679,12 +3703,12 @@ static UniValue DescribeWalletAddress(CWallet* pwallet, const CTxDestination& de UniValue ret(UniValue::VOBJ); UniValue detail = DescribeAddress(dest); CScript script = GetScriptForDestination(dest); - const SigningProvider* provider = nullptr; + std::unique_ptr<SigningProvider> provider = nullptr; if (pwallet) { provider = pwallet->GetSigningProvider(script); } ret.pushKVs(detail); - ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider), dest)); + ret.pushKVs(boost::apply_visitor(DescribeWalletAddressVisitor(provider.get()), dest)); return ret; } @@ -3708,6 +3732,8 @@ UniValue getaddressinfo(const JSONRPCRequest& request) return NullUniValue; } + const std::string example_address = "\"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl\""; + RPCHelpMan{"getaddressinfo", "\nReturn information about the given bitcoin address.\n" "Some of the information will only be present if the address is in the active wallet.\n", @@ -3742,24 +3768,26 @@ UniValue getaddressinfo(const JSONRPCRequest& request) " getaddressinfo output fields for the embedded address, excluding metadata (timestamp, hdkeypath,\n" " hdseedid) and relation to the wallet (ismine, iswatchonly).\n" " \"iscompressed\" : true|false, (boolean, optional) If the pubkey is compressed.\n" - " \"label\" : \"label\" (string) The label associated with the address. Defaults to \"\". Equivalent to the name field in the labels array.\n" + " \"label\" : \"label\" (string) DEPRECATED. The label associated with the address. Defaults to \"\". Replaced by the labels array below.\n" " \"timestamp\" : timestamp, (number, optional) The creation time of the key, if available, expressed in " + UNIX_EPOCH_TIME + ".\n" " \"hdkeypath\" : \"keypath\" (string, optional) The HD keypath, if the key is HD and available.\n" " \"hdseedid\" : \"<hash160>\" (string, optional) The Hash160 of the HD seed.\n" " \"hdmasterfingerprint\" : \"<hash160>\" (string, optional) The fingerprint of the master key.\n" - " \"labels\" (object) An array of labels associated with the address. Currently limited to one label but returned\n" - " as an array to keep the API stable if multiple labels are enabled in the future.\n" + " \"labels\" (array) Array of labels associated with the address. Currently limited to one label but returned\n" + " as an array to keep the API stable if multiple labels are enabled in the future.\n" " [\n" - " { (json object of label data)\n" - " \"name\": \"label name\" (string) The label name. Defaults to \"\". Equivalent to the label field above.\n" - " \"purpose\": \"purpose\" (string) The purpose of the associated address (send or receive).\n" - " },...\n" + " \"label name\" (string) The label name. Defaults to \"\".\n" + " DEPRECATED, will be removed in 0.21. To re-enable, launch bitcoind with `-deprecatedrpc=labelspurpose`:\n" + " {\n" + " \"name\" : \"label name\" (string) The label name. Defaults to \"\".\n" + " \"purpose\" : \"purpose\" (string) The purpose of the associated address (send or receive).\n" + " }\n" " ]\n" "}\n" }, RPCExamples{ - HelpExampleCli("getaddressinfo", "\"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl\"") + - HelpExampleRpc("getaddressinfo", "\"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl\"") + HelpExampleCli("getaddressinfo", example_address) + + HelpExampleRpc("getaddressinfo", example_address) }, }.Check(request); @@ -3767,7 +3795,6 @@ UniValue getaddressinfo(const JSONRPCRequest& request) UniValue ret(UniValue::VOBJ); CTxDestination dest = DecodeDestination(request.params[0].get_str()); - // Make sure the destination is valid if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); @@ -3779,7 +3806,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) CScript scriptPubKey = GetScriptForDestination(dest); ret.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); - const SigningProvider* provider = pwallet->GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = pwallet->GetSigningProvider(scriptPubKey); isminetype mine = pwallet->IsMine(dest); ret.pushKV("ismine", bool(mine & ISMINE_SPENDABLE)); @@ -3793,24 +3820,18 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ret.pushKV("iswatchonly", bool(mine & ISMINE_WATCH_ONLY)); - // Return DescribeWalletAddress fields. - // Always returned: isscript, ischange, iswitness. - // Optional: witness_version, witness_program, script, hex, pubkeys (array), - // sigsrequired, pubkey, embedded, iscompressed. UniValue detail = DescribeWalletAddress(pwallet, dest); ret.pushKVs(detail); - // Return label field if existing. Currently only one label can be - // associated with an address, so the label should be equivalent to the - // value of the name key/value pair in the labels hash array below. - if (pwallet->mapAddressBook.count(dest)) { + // DEPRECATED: Return label field if existing. Currently only one label can + // be associated with an address, so the label should be equivalent to the + // value of the name key/value pair in the labels array below. + if ((pwallet->chain().rpcEnableDeprecated("label")) && (pwallet->mapAddressBook.count(dest))) { ret.pushKV("label", pwallet->mapAddressBook[dest].name); } ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); - // Fetch KeyMetadata, if present, for the timestamp, hdkeypath, hdseedid, - // and hdmasterfingerprint fields. ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); if (spk_man) { if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { @@ -3823,15 +3844,21 @@ UniValue getaddressinfo(const JSONRPCRequest& request) } } - // Return a labels array containing a hash of key/value pairs for the label - // name and address purpose. The name value is equivalent to the label field - // above. Currently only one label can be associated with an address, but we - // return an array so the API remains stable if we allow multiple labels to - // be associated with an address in the future. + // Return a `labels` array containing the label associated with the address, + // equivalent to the `label` field above. Currently only one label can be + // associated with an address, but we return an array so the API remains + // stable if we allow multiple labels to be associated with an address in + // the future. UniValue labels(UniValue::VARR); std::map<CTxDestination, CAddressBookData>::iterator mi = pwallet->mapAddressBook.find(dest); if (mi != pwallet->mapAddressBook.end()) { - labels.push_back(AddressBookDataToJSON(mi->second, true)); + // DEPRECATED: The previous behavior of returning an array containing a + // JSON object of `name` and `purpose` key/value pairs is deprecated. + if (pwallet->chain().rpcEnableDeprecated("labelspurpose")) { + labels.push_back(AddressBookDataToJSON(mi->second, true)); + } else { + labels.push_back(mi->second.name); + } } ret.pushKV("labels", std::move(labels)); @@ -3981,7 +4008,7 @@ UniValue sethdseed(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); @@ -3992,7 +4019,7 @@ UniValue sethdseed(const JSONRPCRequest& request) } auto locked_chain = pwallet->chain().lock(); - LOCK(pwallet->cs_wallet); + LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { @@ -4054,10 +4081,9 @@ UniValue walletprocesspsbt(const JSONRPCRequest& request) {"bip32derivs", RPCArg::Type::BOOL, /* default */ "false", "If true, includes the BIP 32 derivation paths for public keys if we know them"}, }, RPCResult{ - "{\n" - " \"psbt\" : \"value\", (string) The base64-encoded partially signed transaction\n" + "{ (json object)\n" + " \"psbt\" : \"str\", (string) The base64-encoded partially signed transaction\n" " \"complete\" : true|false, (boolean) If the transaction has a complete set of signatures\n" - " ]\n" "}\n" }, RPCExamples{ diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index becca455f6..2813fa2bfc 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -41,7 +41,7 @@ std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& reques std::string HelpRequiringPassphrase(const CWallet*); void EnsureWalletIsUnlocked(const CWallet*); bool EnsureWalletIsAvailable(const CWallet*, bool avoidException); -LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet); +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); UniValue getaddressinfo(const JSONRPCRequest& request); UniValue signrawtransactionwithwallet(const JSONRPCRequest& request); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index be8a71da97..4c9d88973e 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -9,10 +9,10 @@ #include <util/strencodings.h> #include <util/translation.h> #include <wallet/scriptpubkeyman.h> -#include <wallet/wallet.h> bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { + LOCK(cs_KeyStore); error.clear(); // Generate a new key that is added to wallet @@ -238,7 +238,6 @@ bool LegacyScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) { - AssertLockHeld(cs_wallet); LOCK(cs_KeyStore); encrypted_batch = batch; if (!mapCryptedKeys.empty()) { @@ -269,6 +268,7 @@ bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBat bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) { + LOCK(cs_KeyStore); if (!CanGetAddresses(internal)) { return false; } @@ -282,7 +282,7 @@ bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool i void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); // extract addresses and check if they match with an unused keypool key for (const auto& keyid : GetAffectedKeys(script, *this)) { std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); @@ -299,7 +299,7 @@ void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) void LegacyScriptPubKeyMan::UpgradeKeyMetadata() { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); if (m_storage.IsLocked() || m_storage.IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { return; } @@ -352,7 +352,7 @@ bool LegacyScriptPubKeyMan::IsHDEnabled() const bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); // Check if the keypool has keys bool keypool_has_keys; if (internal && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { @@ -369,7 +369,7 @@ bool LegacyScriptPubKeyMan::CanGetAddresses(bool internal) bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); error = ""; bool hd_upgrade = false; bool split_upgrade = false; @@ -383,7 +383,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, std::string& error) hd_upgrade = true; } // Upgrade to HD chain split if necessary - if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) { + if (m_storage.CanSupportFeature(FEATURE_HD_SPLIT) && CHDChain::VERSION_HD_CHAIN_SPLIT) { WalletLogPrintf("Upgrading wallet to use HD chain split\n"); m_storage.SetMinVersion(FEATURE_PRE_SPLIT_KEYPOOL); split_upgrade = FEATURE_HD_SPLIT > prev_version; @@ -410,7 +410,7 @@ bool LegacyScriptPubKeyMan::HavePrivateKeys() const void LegacyScriptPubKeyMan::RewriteDB() { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); setInternalKeyPool.clear(); setExternalKeyPool.clear(); m_pool_key_to_index.clear(); @@ -435,7 +435,7 @@ static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, Walle int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() { - LOCK(cs_wallet); + LOCK(cs_KeyStore); WalletBatch batch(m_storage.GetDatabase()); @@ -453,25 +453,53 @@ int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() size_t LegacyScriptPubKeyMan::KeypoolCountExternalKeys() { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); return setExternalKeyPool.size() + set_pre_split_keypool.size(); } unsigned int LegacyScriptPubKeyMan::GetKeyPoolSize() const { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); return setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(); } int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); return nTimeFirstKey; } +std::unique_ptr<SigningProvider> LegacyScriptPubKeyMan::GetSigningProvider(const CScript& script) const +{ + return MakeUnique<LegacySigningProvider>(*this); +} + +bool LegacyScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sigdata) +{ + if (IsMine(script) != ISMINE_NO) { + // If it IsMine, we can always provide in some way + return true; + } else if (HaveCScript(CScriptID(script))) { + // We can still provide some stuff if we have the script, but IsMine failed because we don't have keys + return true; + } else { + // If, given the stuff in sigdata, we could make a valid sigature, then we can provide for this script + ProduceSignature(*this, DUMMY_SIGNATURE_CREATOR, script, sigdata); + if (!sigdata.signatures.empty()) { + // If we could make signatures, make sure we have a private key to actually make a signature + bool has_privkeys = false; + for (const auto& key_sig_pair : sigdata.signatures) { + has_privkeys |= HaveKey(key_sig_pair.first); + } + return has_privkeys; + } + return false; + } +} + const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); CKeyID key_id = GetKeyForDestination(*this, dest); if (!key_id.IsNull()) { @@ -490,13 +518,18 @@ const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& des return nullptr; } +uint256 LegacyScriptPubKeyMan::GetID() const +{ + return UINT256_ONE(); +} + /** * Update wallet first key creation time. This should be called whenever keys * are added to the wallet, with the oldest key creation time. */ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) { - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); if (nCreateTime <= 1) { // Cannot determine birthday information, so set the wallet birthday to // the beginning of time. @@ -513,13 +546,14 @@ bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) bool LegacyScriptPubKeyMan::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) { + LOCK(cs_KeyStore); WalletBatch batch(m_storage.GetDatabase()); return LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(batch, secret, pubkey); } bool LegacyScriptPubKeyMan::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) { - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); // Make sure we aren't adding private keys to private key disabled wallets assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); @@ -574,14 +608,14 @@ bool LegacyScriptPubKeyMan::LoadCScript(const CScript& redeemScript) void LegacyScriptPubKeyMan::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); UpdateTimeFirstKey(meta.nCreateTime); mapKeyMetadata[keyID] = meta; } void LegacyScriptPubKeyMan::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); UpdateTimeFirstKey(meta.nCreateTime); m_script_metadata[script_id] = meta; } @@ -630,7 +664,7 @@ bool LegacyScriptPubKeyMan::AddCryptedKey(const CPubKey &vchPubKey, if (!AddCryptedKeyInner(vchPubKey, vchCryptedSecret)) return false; { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (encrypted_batch) return encrypted_batch->WriteCryptedKey(vchPubKey, vchCryptedSecret, @@ -663,7 +697,6 @@ static bool ExtractPubKey(const CScript &dest, CPubKey& pubKeyOut) bool LegacyScriptPubKeyMan::RemoveWatchOnly(const CScript &dest) { - AssertLockHeld(cs_wallet); { LOCK(cs_KeyStore); setWatchOnly.erase(dest); @@ -734,7 +767,7 @@ bool LegacyScriptPubKeyMan::AddWatchOnly(const CScript& dest, int64_t nCreateTim void LegacyScriptPubKeyMan::SetHDChain(const CHDChain& chain, bool memonly) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (!memonly && !WalletBatch(m_storage.GetDatabase()).WriteHDChain(chain)) throw std::runtime_error(std::string(__func__) + ": writing chain failed"); @@ -771,7 +804,7 @@ bool LegacyScriptPubKeyMan::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& inf { CKeyMetadata meta; { - LOCK(cs_wallet); + LOCK(cs_KeyStore); auto it = mapKeyMetadata.find(keyID); if (it != mapKeyMetadata.end()) { meta = it->second; @@ -821,7 +854,7 @@ CPubKey LegacyScriptPubKeyMan::GenerateNewKey(WalletBatch &batch, bool internal) { assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); bool fCompressed = m_storage.CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets CKey secret; @@ -913,7 +946,7 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) { - AssertLockHeld(cs_wallet); + LOCK(cs_KeyStore); if (keypool.m_pre_split) { set_pre_split_keypool.insert(nIndex); } else if (keypool.fInternal) { @@ -935,7 +968,7 @@ void LegacyScriptPubKeyMan::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) bool LegacyScriptPubKeyMan::CanGenerateKeys() { // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) - LOCK(cs_wallet); + LOCK(cs_KeyStore); return IsHDEnabled() || !m_storage.CanSupportFeature(FEATURE_HD); } @@ -962,7 +995,7 @@ CPubKey LegacyScriptPubKeyMan::DeriveNewSeed(const CKey& key) metadata.hd_seed_id = seed.GetID(); { - LOCK(cs_wallet); + LOCK(cs_KeyStore); // mem store the metadata mapKeyMetadata[seed.GetID()] = metadata; @@ -977,7 +1010,7 @@ CPubKey LegacyScriptPubKeyMan::DeriveNewSeed(const CKey& key) void LegacyScriptPubKeyMan::SetHDSeed(const CPubKey& seed) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object @@ -1000,7 +1033,7 @@ bool LegacyScriptPubKeyMan::NewKeyPool() return false; } { - LOCK(cs_wallet); + LOCK(cs_KeyStore); WalletBatch batch(m_storage.GetDatabase()); for (const int64_t nIndex : setInternalKeyPool) { @@ -1034,7 +1067,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) return false; } { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (m_storage.IsLocked()) return false; @@ -1076,7 +1109,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) void LegacyScriptPubKeyMan::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? int64_t index = ++m_max_keypool_index; if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { @@ -1107,7 +1140,7 @@ void LegacyScriptPubKeyMan::ReturnDestination(int64_t nIndex, bool fInternal, co { // Return to key pool { - LOCK(cs_wallet); + LOCK(cs_KeyStore); if (fInternal) { setInternalKeyPool.insert(nIndex); } else if (!set_pre_split_keypool.empty()) { @@ -1131,7 +1164,7 @@ bool LegacyScriptPubKeyMan::GetKeyFromPool(CPubKey& result, const OutputType typ CKeyPool keypool; { - LOCK(cs_wallet); + LOCK(cs_KeyStore); int64_t nIndex; if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (m_storage.IsLocked()) return false; @@ -1150,7 +1183,7 @@ bool LegacyScriptPubKeyMan::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& key nIndex = -1; keypool.vchPubKey = CPubKey(); { - LOCK(cs_wallet); + LOCK(cs_KeyStore); bool fReturningInternal = fRequestedInternal; fReturningInternal &= (IsHDEnabled() && m_storage.CanSupportFeature(FEATURE_HD_SPLIT)) || m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -1210,7 +1243,7 @@ void LegacyScriptPubKeyMan::LearnAllRelatedScripts(const CPubKey& key) void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) { - AssertLockHeld(cs_wallet); + AssertLockHeld(cs_KeyStore); bool internal = setInternalKeyPool.count(keypool_id); if (!internal) assert(setExternalKeyPool.count(keypool_id) || set_pre_split_keypool.count(keypool_id)); std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); @@ -1281,7 +1314,7 @@ bool LegacyScriptPubKeyMan::AddCScriptWithDB(WalletBatch& batch, const CScript& bool LegacyScriptPubKeyMan::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey& pubkey, const KeyOriginInfo& info) { - LOCK(cs_wallet); + LOCK(cs_KeyStore); std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; mapKeyMetadata[pubkey.GetID()].has_key_origin = true; @@ -1393,13 +1426,3 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const } return set_address; } - -// Temporary CWallet accessors and aliases. -LegacyScriptPubKeyMan::LegacyScriptPubKeyMan(CWallet& wallet) - : ScriptPubKeyMan(wallet), - m_wallet(wallet), - cs_wallet(wallet.cs_wallet) {} - -void LegacyScriptPubKeyMan::NotifyWatchonlyChanged(bool fHaveWatchOnly) const { return m_wallet.NotifyWatchonlyChanged(fHaveWatchOnly); } -void LegacyScriptPubKeyMan::NotifyCanGetAddressesChanged() const { return m_wallet.NotifyCanGetAddressesChanged(); } -template<typename... Params> void LegacyScriptPubKeyMan::WalletLogPrintf(const std::string& fmt, const Params&... parameters) const { return m_wallet.WalletLogPrintf(fmt, parameters...); } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index b78494921c..7b1c023bc9 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -74,6 +74,11 @@ std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& p * keys (by default 1000) ahead of the last used key and scans for the * addresses of those keys. This avoids the risk of not seeing transactions * involving the wallet's addresses, or of re-using the same address. + * In the unlikely case where none of the addresses in the `gap limit` are + * used on-chain, the look-ahead will not be incremented to keep + * a constant size and addresses beyond this range will not be detected by an + * old backup. For this reason, it is not recommended to decrease keypool size + * lower than default value. * * The HD-split wallet feature added a second keypool (commit: 02592f4c). There * is an external keypool (for addresses to hand out) and an internal keypool @@ -196,8 +201,28 @@ public: virtual int64_t GetTimeFirstKey() const { return 0; } - //! Return address metadata virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } + + virtual std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const { return nullptr; } + + /** Whether this ScriptPubKeyMan can provide a SigningProvider (via GetSigningProvider) that, combined with + * sigdata, can produce a valid signature. + */ + virtual bool CanProvide(const CScript& script, SignatureData& sigdata) { return false; } + + virtual uint256 GetID() const { return uint256(); } + + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ + template<typename... Params> + void WalletLogPrintf(std::string fmt, Params... parameters) const { + LogPrintf(("%s " + fmt).c_str(), m_storage.GetDisplayName(), parameters...); + }; + + /** Watch-only address added */ + boost::signals2::signal<void (bool fHaveWatchOnly)> NotifyWatchonlyChanged; + + /** Keypool has new keys */ + boost::signals2::signal<void ()> NotifyCanGetAddressesChanged; }; class LegacyScriptPubKeyMan : public ScriptPubKeyMan, public FillableSigningProvider @@ -209,7 +234,7 @@ private: using WatchOnlySet = std::set<CScript>; using WatchKeyMap = std::map<CKeyID, CPubKey>; - WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; + WalletBatch *encrypted_batch GUARDED_BY(cs_KeyStore) = nullptr; using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; @@ -217,7 +242,7 @@ private: WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); - int64_t nTimeFirstKey GUARDED_BY(cs_wallet) = 0; + int64_t nTimeFirstKey GUARDED_BY(cs_KeyStore) = 0; bool AddKeyPubKeyInner(const CKey& key, const CPubKey &pubkey); bool AddCryptedKeyInner(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); @@ -231,14 +256,14 @@ private: * of the other AddWatchOnly which accepts a timestamp and sets * nTimeFirstKey more intelligently for more efficient rescans. */ - bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnly(const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); bool AddWatchOnlyInMem(const CScript &dest); //! Adds a watch-only address to the store, and saves it to disk. - bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript& dest, int64_t create_time) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddKeyPubKeyWithDB(WalletBatch &batch,const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); @@ -252,12 +277,12 @@ private: CHDChain hdChain; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); - std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); - int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_KeyStore); + std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_KeyStore); + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_KeyStore); + int64_t m_max_keypool_index GUARDED_BY(cs_KeyStore) = 0; std::map<CKeyID, int64_t> m_pool_key_to_index; // Tracks keypool indexes to CKeyIDs of keys that have been taken out of the keypool but may be returned to it std::map<int64_t, CKeyID> m_index_to_reserved_key; @@ -282,6 +307,8 @@ private: bool ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal); public: + using ScriptPubKeyMan::ScriptPubKeyMan; + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; isminetype IsMine(const CScript& script) const override; @@ -297,7 +324,7 @@ public: void MarkUnusedAddresses(const CScript& script) override; //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo - void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void UpgradeKeyMetadata(); bool IsHDEnabled() const override; @@ -310,7 +337,7 @@ public: void RewriteDB() override; int64_t GetOldestKeyPoolTime() override; - size_t KeypoolCountExternalKeys() override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + size_t KeypoolCountExternalKeys() override; unsigned int GetKeyPoolSize() const override; int64_t GetTimeFirstKey() const override; @@ -319,28 +346,34 @@ public: bool CanGetAddresses(bool internal = false) override; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const override; + + bool CanProvide(const CScript& script, SignatureData& sigdata) override; + + uint256 GetID() const override; + // Map from Key ID to key metadata. - std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_wallet); + std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); // Map from Script ID to key metadata (for watch-only keys). - std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); + std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_KeyStore); //! Adds a key to the store, and saves it to disk. - bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; //! Adds a key to the store, without saving it to disk (used by LoadWallet) bool LoadKey(const CKey& key, const CPubKey &pubkey); //! Adds an encrypted key to the store, and saves it to disk. bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); //! Adds an encrypted key to the store, without saving it to disk (used by LoadWallet) bool LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); - void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Adds a CScript to the store bool LoadCScript(const CScript& redeemScript); //! Load metadata (used by LoadWallet) - void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); + void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); //! Generate a new key - CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + CPubKey GenerateNewKey(WalletBatch& batch, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); /* Set the HD chain model (chain child index counters) */ void SetHDChain(const CHDChain& chain, bool memonly); @@ -353,8 +386,8 @@ public: //! Returns whether there are any watch-only things in the wallet bool HaveWatchOnly() const; //! Remove a watch only script from the keystore - bool RemoveWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool RemoveWatchOnly(const CScript &dest); + bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); //! Fetches a pubkey from mapWatchKeys if it exists there bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const; @@ -367,14 +400,14 @@ public: bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; //! Load a keypool entry - void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool); bool NewKeyPool(); - void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void MarkPreSplitKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); - bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + bool ImportScriptPubKeys(const std::set<CScript>& script_pub_keys, const bool have_solving_data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); /* Returns true if the wallet can generate new keys */ bool CanGenerateKeys(); @@ -408,19 +441,26 @@ public: /** * Marks all keys in the keypool up to and including reserve_key as used. */ - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set<CKeyID> GetKeys() const override; - // Temporary CWallet accessors and aliases. - friend class CWallet; - friend class ReserveDestination; - LegacyScriptPubKeyMan(CWallet& wallet); - void NotifyWatchonlyChanged(bool fHaveWatchOnly) const; - void NotifyCanGetAddressesChanged() const; - template<typename... Params> void WalletLogPrintf(const std::string& fmt, const Params&... parameters) const; - CWallet& m_wallet; - CCriticalSection& cs_wallet; +}; + +/** Wraps a LegacyScriptPubKeyMan so that it can be returned in a new unique_ptr */ +class LegacySigningProvider : public SigningProvider +{ +private: + const LegacyScriptPubKeyMan& m_spk_man; +public: + LegacySigningProvider(const LegacyScriptPubKeyMan& spk_man) : m_spk_man(spk_man) {} + + bool GetCScript(const CScriptID &scriptid, CScript& script) const override { return m_spk_man.GetCScript(scriptid, script); } + bool HaveCScript(const CScriptID &scriptid) const override { return m_spk_man.HaveCScript(scriptid); } + bool GetPubKey(const CKeyID &address, CPubKey& pubkey) const override { return m_spk_man.GetPubKey(address, pubkey); } + bool GetKey(const CKeyID &address, CKey& key) const override { return m_spk_man.GetKey(address, key); } + bool HaveKey(const CKeyID &address) const override { return m_spk_man.HaveKey(address); } + bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override { return m_spk_man.GetKeyOrigin(keyid, info); } }; #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 7e9f88f6b7..d65a0e9075 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -78,6 +78,7 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo if (fIsFromMe) { wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); + wtx->m_is_cache_empty = false; } COutput output(wtx.get(), nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); @@ -135,6 +136,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) { LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); // Setup std::vector<CInputCoin> utxo_pool; @@ -277,6 +279,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); bool firstRun; wallet->LoadWallet(firstRun); + wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); add_coin(*wallet, 5 * CENT, 6 * 24, false, 0, true); add_coin(*wallet, 3 * CENT, 6 * 24, false, 0, true); @@ -298,6 +301,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) bool bnb_used; LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) @@ -577,6 +581,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) bool bnb_used; LOCK(testWallet.cs_wallet); + testWallet.SetupLegacyScriptPubKeyMan(); empty_wallet(); @@ -595,6 +600,8 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { + testWallet.SetupLegacyScriptPubKeyMan(); + // Random generator stuff std::default_random_engine generator; std::exponential_distribution<double> distribution (100); diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 86ba0013fe..797a0d634f 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 76c3639d16..4c0e4dc653 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -36,7 +36,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); // Keystore does not have key @@ -52,7 +53,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); // Keystore does not have key @@ -68,7 +70,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); // Keystore does not have key @@ -84,7 +87,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); // Keystore does not have key @@ -100,7 +104,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemScript = GetScriptForDestination(PKHash(pubkeys[0])); scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); @@ -123,7 +128,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemscript_inner = GetScriptForDestination(PKHash(pubkeys[0])); CScript redeemscript = GetScriptForDestination(ScriptHash(redeemscript_inner)); @@ -140,7 +146,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript redeemscript = GetScriptForDestination(PKHash(pubkeys[0])); CScript witnessscript = GetScriptForDestination(ScriptHash(redeemscript)); @@ -157,7 +164,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessscript = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessscript)); @@ -172,7 +180,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessscript_inner = GetScriptForDestination(PKHash(pubkeys[0])); CScript witnessscript = GetScriptForDestination(WitnessV0ScriptHash(witnessscript_inner)); @@ -189,7 +198,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(pubkeys[0]))); @@ -203,7 +213,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(PKHash(uncompressedPubkey))); @@ -221,7 +232,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); @@ -251,7 +263,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); @@ -271,7 +284,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); @@ -296,7 +310,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[1])); @@ -321,7 +336,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); CScript witnessScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); CScript redeemScript = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); @@ -347,7 +363,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); @@ -360,7 +377,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); @@ -373,7 +391,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); @@ -386,7 +405,8 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { CWallet keystore(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - LOCK(keystore.cs_wallet); + keystore.SetupLegacyScriptPubKeyMan(); + LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); scriptPubKey.clear(); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index d930ca6bea..f923de6178 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -16,8 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) BOOST_AUTO_TEST_CASE(psbt_updater_test) { - auto spk_man = m_wallet.GetLegacyScriptPubKeyMan(); - LOCK(m_wallet.cs_wallet); + auto spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(m_wallet.cs_wallet, spk_man->cs_KeyStore); // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); @@ -68,6 +68,15 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) ssTx << psbtx; std::string final_hex = HexStr(ssTx.begin(), ssTx.end()); BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); + + // Mutate the transaction so that one of the inputs is invalid + psbtx.tx->vin[0].prevout.n = 2; + + // Try to sign the mutated input + SignatureData sigdata; + psbtx.inputs[0].FillSignatureData(sigdata); + const std::unique_ptr<SigningProvider> provider = m_wallet.GetSigningProvider(ws1, sigdata); + BOOST_CHECK(!SignPSBTInput(*provider, psbtx, 0, SIGHASH_ALL)); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 2f21b2439b..a487e9e2e0 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -28,9 +28,8 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static void AddKey(CWallet& wallet, const CKey& key) { - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK(wallet.cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); spk_man->AddKeyPubKey(key, key.GetPubKey()); } @@ -152,6 +151,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // after. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -216,9 +216,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - auto spk_man = wallet->GetLegacyScriptPubKeyMan(); - LOCK(wallet->cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); @@ -234,6 +233,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // were scanned, and no prior blocks were scanned. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); JSONRPCRequest request; request.params.setArray(); @@ -267,13 +267,12 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) auto chain = interfaces::MakeChain(node); CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); + auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); - LOCK(wallet.cs_wallet); - AssertLockHeld(spk_man->cs_wallet); + LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, ::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash(), 0); @@ -283,7 +282,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // cache the current immature credit amount, which is 0. BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); - // Invalidate the cached vanue, add the key, and make sure a new immature + // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); @@ -377,7 +376,7 @@ static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& a CScript p2pk = GetScriptForRawPubKey(add_pubkey); CKeyID add_address = add_pubkey.GetID(); CPubKey found_pubkey; - LOCK(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); // all Scripts (i.e. also all PubKeys) are added to the general watch-only set BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); @@ -394,7 +393,6 @@ static void TestWatchOnlyPubKey(LegacyScriptPubKeyMan* spk_man, const CPubKey& a BOOST_CHECK(found_pubkey == CPubKey()); // passed key is unchanged } - AssertLockHeld(spk_man->cs_wallet); spk_man->RemoveWatchOnly(p2pk); BOOST_CHECK(!spk_man->HaveWatchOnly(p2pk)); @@ -419,7 +417,7 @@ BOOST_AUTO_TEST_CASE(WatchOnlyPubKeys) { CKey key; CPubKey pubkey; - LegacyScriptPubKeyMan* spk_man = m_wallet.GetLegacyScriptPubKeyMan(); + LegacyScriptPubKeyMan* spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); BOOST_CHECK(!spk_man->HaveWatchOnly()); @@ -581,6 +579,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 3954f66267..405afb6d8d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,7 @@ #include <interfaces/wallet.h> #include <key.h> #include <key_io.h> +#include <optional.h> #include <policy/fees.h> #include <policy/policy.h> #include <primitives/block.h> @@ -45,8 +46,9 @@ const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ static const size_t OUTPUT_GROUP_MAX_ENTRIES = 10; -static CCriticalSection cs_wallets; +static RecursiveMutex cs_wallets; static std::vector<std::shared_ptr<CWallet>> vpwallets GUARDED_BY(cs_wallets); +static std::list<LoadWalletFn> g_load_wallet_fns GUARDED_BY(cs_wallets); bool AddWallet(const std::shared_ptr<CWallet>& wallet) { @@ -55,6 +57,7 @@ bool AddWallet(const std::shared_ptr<CWallet>& wallet) std::vector<std::shared_ptr<CWallet>>::const_iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); if (i != vpwallets.end()) return false; vpwallets.push_back(wallet); + wallet->ConnectScriptPubKeyManNotifiers(); return true; } @@ -89,6 +92,13 @@ std::shared_ptr<CWallet> GetWallet(const std::string& name) return nullptr; } +std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) +{ + LOCK(cs_wallets); + auto it = g_load_wallet_fns.emplace(g_load_wallet_fns.end(), std::move(load_wallet)); + return interfaces::MakeHandler([it] { LOCK(cs_wallets); g_load_wallet_fns.erase(it); }); +} + static Mutex g_wallet_release_mutex; static std::condition_variable g_wallet_release_cv; static std::set<std::string> g_unloading_wallet_set; @@ -211,7 +221,8 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& // Set a seed for the wallet { - if (auto spk_man = wallet->m_spk_man.get()) { + LOCK(wallet->cs_wallet); + for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { error = "Unable to generate initial keys"; return WalletCreationStatus::CREATION_FAILED; @@ -229,7 +240,7 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& return WalletCreationStatus::SUCCESS; } -const uint256 CWalletTx::ABANDON_HASH(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); +const uint256 CWalletTx::ABANDON_HASH(UINT256_ONE()); /** @defgroup mapWallet * @@ -256,10 +267,12 @@ void CWallet::UpgradeKeyMetadata() return; } - if (m_spk_man) { - AssertLockHeld(m_spk_man->cs_wallet); - m_spk_man->UpgradeKeyMetadata(); + auto spk_man = GetLegacyScriptPubKeyMan(); + if (!spk_man) { + return; } + + spk_man->UpgradeKeyMetadata(); SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } @@ -540,7 +553,8 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) } encrypted_batch->WriteMasterKey(nMasterKeyMaxID, kMasterKey); - if (auto spk_man = m_spk_man.get()) { + for (const auto& spk_man_pair : m_spk_managers) { + auto spk_man = spk_man_pair.second.get(); if (!spk_man->Encrypt(_vMasterKey, encrypted_batch)) { encrypted_batch->TxnAbort(); delete encrypted_batch; @@ -569,7 +583,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Unlock(strWalletPassphrase); // if we are using HD, replace the HD seed with a new one - if (auto spk_man = m_spk_man.get()) { + if (auto spk_man = GetLegacyScriptPubKeyMan()) { if (spk_man->IsHDEnabled()) { if (!spk_man->SetupGeneration(true)) { return false; @@ -700,7 +714,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) return success; } -void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used) +void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) { AssertLockHeld(cs_wallet); const CWalletTx* srctx = GetWalletTx(hash); @@ -710,7 +724,9 @@ void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, u if (ExtractDestination(srctx->tx->vout[n].scriptPubKey, dst)) { if (IsMine(dst)) { if (used && !GetDestData(dst, "used", nullptr)) { - AddDestData(batch, dst, "used", "p"); // p for "present", opposite of absent (null) + if (AddDestData(batch, dst, "used", "p")) { // p for "present", opposite of absent (null) + tx_destinations.insert(dst); + } } else if (!used && GetDestData(dst, "used", nullptr)) { EraseDestData(batch, dst, "used"); } @@ -718,17 +734,33 @@ void CWallet::SetUsedDestinationState(WalletBatch& batch, const uint256& hash, u } } -bool CWallet::IsUsedDestination(const CTxDestination& dst) const -{ - LOCK(cs_wallet); - return IsMine(dst) && GetDestData(dst, "used", nullptr); -} - bool CWallet::IsUsedDestination(const uint256& hash, unsigned int n) const { + AssertLockHeld(cs_wallet); CTxDestination dst; const CWalletTx* srctx = GetWalletTx(hash); - return srctx && ExtractDestination(srctx->tx->vout[n].scriptPubKey, dst) && IsUsedDestination(dst); + if (srctx) { + assert(srctx->tx->vout.size() > n); + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + // When descriptor wallets arrive, these additional checks are + // likely superfluous and can be optimized out + assert(spk_man != nullptr); + for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { + WitnessV0KeyHash wpkh_dest(keyid); + if (GetDestData(wpkh_dest, "used", nullptr)) { + return true; + } + ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); + if (GetDestData(sh_wpkh_dest, "used", nullptr)) { + return true; + } + PKHash pkh_dest(keyid); + if (GetDestData(pkh_dest, "used", nullptr)) { + return true; + } + } + } + return false; } bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) @@ -741,10 +773,14 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) if (IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)) { // Mark used destinations + std::set<CTxDestination> tx_destinations; + for (const CTxIn& txin : wtxIn.tx->vin) { const COutPoint& op = txin.prevout; - SetUsedDestinationState(batch, op.hash, op.n, true); + SetUsedDestinationState(batch, op.hash, op.n, true, tx_destinations); } + + MarkDestinationsDirty(tx_destinations); } // Inserts only if not already there, returns tx inserted or tx found @@ -892,8 +928,8 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co // loop though all outputs for (const CTxOut& txout: tx.vout) { - if (auto spk_man = m_spk_man.get()) { - spk_man->MarkUnusedAddresses(txout.scriptPubKey); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->MarkUnusedAddresses(txout.scriptPubKey); } } @@ -1164,8 +1200,8 @@ isminetype CWallet::IsMine(const CTxDestination& dest) const isminetype CWallet::IsMine(const CScript& script) const { isminetype result = ISMINE_NO; - if (auto spk_man = m_spk_man.get()) { - result = spk_man->IsMine(script); + for (const auto& spk_man_pair : m_spk_managers) { + result = std::max(result, spk_man_pair.second->IsMine(script)); } return result; } @@ -1284,16 +1320,18 @@ CAmount CWallet::GetChange(const CTransaction& tx) const bool CWallet::IsHDEnabled() const { bool result = true; - if (auto spk_man = m_spk_man.get()) { - result &= spk_man->IsHDEnabled(); + for (const auto& spk_man_pair : m_spk_managers) { + result &= spk_man_pair.second->IsHDEnabled(); } return result; } bool CWallet::CanGetAddresses(bool internal) { - { - auto spk_man = m_spk_man.get(); + LOCK(cs_wallet); + if (m_spk_managers.empty()) return false; + for (OutputType t : OUTPUT_TYPES) { + auto spk_man = GetScriptPubKeyMan(t, internal); if (spk_man && spk_man->CanGetAddresses(internal)) { return true; } @@ -1362,7 +1400,7 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - const SigningProvider* provider = GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); if (!provider) { // We don't know about this scriptpbuKey; return false; @@ -1397,7 +1435,7 @@ bool CWallet::ImportScripts(const std::set<CScript> scripts, int64_t timestamp) if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); return spk_man->ImportScripts(scripts, timestamp); } @@ -1407,7 +1445,7 @@ bool CWallet::ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const in if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); return spk_man->ImportPrivKeys(privkey_map, timestamp); } @@ -1417,7 +1455,7 @@ bool CWallet::ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const st if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); return spk_man->ImportPubKeys(ordered_pubkeys, pubkey_map, key_origins, add_keypool, internal, timestamp); } @@ -1427,7 +1465,7 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri if (!spk_man) { return false; } - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); if (!spk_man->ImportScriptPubKeys(script_pub_keys, have_solving_data, timestamp)) { return false; } @@ -1765,6 +1803,7 @@ CAmount CWalletTx::GetCachableAmount(AmountType type, const isminefilter& filter auto& amount = m_amounts[type]; if (recalculate || !amount.m_cached[filter]) { amount.Set(filter, type == DEBIT ? pwallet->GetDebit(*tx, filter) : pwallet->GetCredit(*tx, filter)); + m_is_cache_empty = false; } return amount.m_value[filter]; } @@ -1841,6 +1880,7 @@ CAmount CWalletTx::GetAvailableCredit(bool fUseCache, const isminefilter& filter if (allow_cache) { m_amounts[AVAILABLE_CREDIT].Set(filter, nCredit); + m_is_cache_empty = false; } return nCredit; @@ -2124,7 +2164,7 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< continue; } - const SigningProvider* provider = GetSigningProvider(wtx.tx->vout[i].scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(wtx.tx->vout[i].scriptPubKey); bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); @@ -2167,8 +2207,8 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins(interfaces::Ch std::vector<COutPoint> lockedCoins; ListLockedCoins(lockedCoins); - // Include watch-only for wallets without private keys - const bool include_watch_only = IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + // Include watch-only for LegacyScriptPubKeyMan wallets without private keys + const bool include_watch_only = GetLegacyScriptPubKeyMan() && IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); const isminetype is_mine_filter = include_watch_only ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; for (const COutPoint& output : lockedCoins) { auto it = mapWallet.find(output.hash); @@ -2378,7 +2418,7 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const CAmount& amount = mi->second.tx->vout[input.prevout.n].nValue; SignatureData sigdata; - const SigningProvider* provider = GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); if (!provider) { // We don't know about this scriptpbuKey; return false; @@ -2847,7 +2887,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - const SigningProvider* provider = GetSigningProvider(scriptPubKey); + std::unique_ptr<SigningProvider> provider = GetSigningProvider(scriptPubKey); if (!provider || !ProduceSignature(*provider, MutableTransactionSignatureCreator(&txNew, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed").translated; @@ -2954,17 +2994,17 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { if (database->Rewrite("\x04pool")) { - if (auto spk_man = m_spk_man.get()) { - spk_man->RewriteDB(); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); } } } - { - LOCK(cs_KeyStore); - // This wallet is in its first run if all of these are empty - fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() - && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); + // This wallet is in its first run if there are no ScriptPubKeyMans and it isn't blank or no privkeys + fFirstRunRet = m_spk_managers.empty() && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); + if (fFirstRunRet) { + assert(m_external_spk_managers.empty()); + assert(m_internal_spk_managers.empty()); } if (nLoadWalletRet != DBErrors::LOAD_OK) @@ -2988,8 +3028,8 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { if (database->Rewrite("\x04pool")) { - if (auto spk_man = m_spk_man.get()) { - spk_man->RewriteDB(); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); } } } @@ -3009,8 +3049,8 @@ DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { if (database->Rewrite("\x04pool")) { - if (auto spk_man = m_spk_man.get()) { - spk_man->RewriteDB(); + for (const auto& spk_man_pair : m_spk_managers) { + spk_man_pair.second->RewriteDB(); } } } @@ -3070,8 +3110,7 @@ size_t CWallet::KeypoolCountExternalKeys() AssertLockHeld(cs_wallet); unsigned int count = 0; - if (auto spk_man = m_spk_man.get()) { - AssertLockHeld(spk_man->cs_wallet); + for (auto spk_man : GetActiveScriptPubKeyMans()) { count += spk_man->KeypoolCountExternalKeys(); } @@ -3083,7 +3122,7 @@ unsigned int CWallet::GetKeyPoolSize() const AssertLockHeld(cs_wallet); unsigned int count = 0; - if (auto spk_man = m_spk_man.get()) { + for (auto spk_man : GetActiveScriptPubKeyMans()) { count += spk_man->GetKeyPoolSize(); } return count; @@ -3091,8 +3130,9 @@ unsigned int CWallet::GetKeyPoolSize() const bool CWallet::TopUpKeyPool(unsigned int kpSize) { + LOCK(cs_wallet); bool res = true; - if (auto spk_man = m_spk_man.get()) { + for (auto spk_man : GetActiveScriptPubKeyMans()) { res &= spk_man->TopUp(kpSize); } return res; @@ -3103,7 +3143,7 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, LOCK(cs_wallet); error.clear(); bool result = false; - auto spk_man = m_spk_man.get(); + auto spk_man = GetScriptPubKeyMan(type, false /* internal */); if (spk_man) { spk_man->TopUp(); result = spk_man->GetNewDestination(type, dest, error); @@ -3117,6 +3157,7 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error) { + LOCK(cs_wallet); error.clear(); ReserveDestination reservedest(this, type); @@ -3131,13 +3172,28 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des int64_t CWallet::GetOldestKeyPoolTime() { + LOCK(cs_wallet); int64_t oldestKey = std::numeric_limits<int64_t>::max(); - if (auto spk_man = m_spk_man.get()) { - oldestKey = spk_man->GetOldestKeyPoolTime(); + for (const auto& spk_man_pair : m_spk_managers) { + oldestKey = std::min(oldestKey, spk_man_pair.second->GetOldestKeyPoolTime()); } return oldestKey; } +void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations) { + for (auto& entry : mapWallet) { + CWalletTx& wtx = entry.second; + if (wtx.m_is_cache_empty) continue; + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { + CTxDestination dst; + if (ExtractDestination(wtx.tx->vout[i].scriptPubKey, dst) && destinations.count(dst)) { + wtx.MarkDirty(); + break; + } + } + } +} + std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain::Lock& locked_chain) { std::map<CTxDestination, CAmount> balances; @@ -3288,7 +3344,7 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal) { - m_spk_man = pwallet->GetLegacyScriptPubKeyMan(); + m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); if (!m_spk_man) { return false; } @@ -3370,7 +3426,7 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); assert(spk_man != nullptr); - AssertLockHeld(spk_man->cs_wallet); + LOCK(spk_man->cs_KeyStore); // get birth times for keys with metadata for (const auto& entry : spk_man->mapKeyMetadata) { @@ -3665,7 +3721,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } - if (auto spk_man = walletInstance->m_spk_man.get()) { + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (!spk_man->Upgrade(prev_version, error)) { return nullptr; } @@ -3678,8 +3734,13 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetMinVersion(FEATURE_LATEST); walletInstance->SetWalletFlags(wallet_creation_flags, false); + + // Always create LegacyScriptPubKeyMan for now + walletInstance->SetupLegacyScriptPubKeyMan(); + if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { - if (auto spk_man = walletInstance->m_spk_man.get()) { + LOCK(walletInstance->cs_wallet); + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { error = _("Unable to generate initial keys").translated; return nullptr; @@ -3694,9 +3755,10 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, error = strprintf(_("Error loading %s: Private keys can only be disabled during creation").translated, walletFile); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - if (walletInstance->m_spk_man) { - if (walletInstance->m_spk_man->HavePrivateKeys()) { + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (spk_man->HavePrivateKeys()) { warnings.push_back(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys").translated, walletFile)); + break; } } } @@ -3849,8 +3911,9 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - Optional<int64_t> time_first_key; - if (auto spk_man = walletInstance->m_spk_man.get()) { + // The way the 'time_first_key' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. + Optional<int64_t> time_first_key = MakeOptional(false, int64_t());; + for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { int64_t time = spk_man->GetTimeFirstKey(); if (!time_first_key || time < *time_first_key) time_first_key = time; } @@ -3895,7 +3958,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - chain.loadWallet(interfaces::MakeWallet(walletInstance)); + { + LOCK(cs_wallets); + for (auto& load_wallet : g_load_wallet_fns) { + load_wallet(interfaces::MakeWallet(walletInstance)); + } + } // Register with the validation interface. It's ok to do this after rescan since we're still holding locked_chain. walletInstance->handleNotifications(); @@ -4013,7 +4081,7 @@ bool CWallet::IsLocked() const if (!IsCrypted()) { return false; } - LOCK(cs_KeyStore); + LOCK(cs_wallet); return vMasterKey.empty(); } @@ -4023,7 +4091,7 @@ bool CWallet::Lock() return false; { - LOCK(cs_KeyStore); + LOCK(cs_wallet); vMasterKey.clear(); } @@ -4034,9 +4102,9 @@ bool CWallet::Lock() bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) { { - LOCK(cs_KeyStore); - if (m_spk_man) { - if (!m_spk_man->CheckDecryptionKey(vMasterKeyIn, accept_no_keys)) { + LOCK(cs_wallet); + for (const auto& spk_man_pair : m_spk_managers) { + if (!spk_man_pair.second->CheckDecryptionKey(vMasterKeyIn, accept_no_keys)) { return false; } } @@ -4046,24 +4114,102 @@ bool CWallet::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys) return true; } +std::set<ScriptPubKeyMan*> CWallet::GetActiveScriptPubKeyMans() const +{ + std::set<ScriptPubKeyMan*> spk_mans; + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_man = GetScriptPubKeyMan(t, internal); + if (spk_man) { + spk_mans.insert(spk_man); + } + } + } + return spk_mans; +} + +std::set<ScriptPubKeyMan*> CWallet::GetAllScriptPubKeyMans() const +{ + std::set<ScriptPubKeyMan*> spk_mans; + for (const auto& spk_man_pair : m_spk_managers) { + spk_mans.insert(spk_man_pair.second.get()); + } + return spk_mans; +} + +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const OutputType& type, bool internal) const +{ + const std::map<OutputType, ScriptPubKeyMan*>& spk_managers = internal ? m_internal_spk_managers : m_external_spk_managers; + std::map<OutputType, ScriptPubKeyMan*>::const_iterator it = spk_managers.find(type); + if (it == spk_managers.end()) { + WalletLogPrintf("%s scriptPubKey Manager for output type %d does not exist\n", internal ? "Internal" : "External", static_cast<int>(type)); + return nullptr; + } + return it->second; +} + ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const CScript& script) const { - return m_spk_man.get(); + SignatureData sigdata; + for (const auto& spk_man_pair : m_spk_managers) { + if (spk_man_pair.second->CanProvide(script, sigdata)) { + return spk_man_pair.second.get(); + } + } + return nullptr; +} + +ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const uint256& id) const +{ + if (m_spk_managers.count(id) > 0) { + return m_spk_managers.at(id).get(); + } + return nullptr; } -const SigningProvider* CWallet::GetSigningProvider(const CScript& script) const +std::unique_ptr<SigningProvider> CWallet::GetSigningProvider(const CScript& script) const { - return m_spk_man.get(); + SignatureData sigdata; + return GetSigningProvider(script, sigdata); } -const SigningProvider* CWallet::GetSigningProvider(const CScript& script, SignatureData& sigdata) const +std::unique_ptr<SigningProvider> CWallet::GetSigningProvider(const CScript& script, SignatureData& sigdata) const { - return m_spk_man.get(); + for (const auto& spk_man_pair : m_spk_managers) { + if (spk_man_pair.second->CanProvide(script, sigdata)) { + return spk_man_pair.second->GetSigningProvider(script); + } + } + return nullptr; } LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { - return m_spk_man.get(); + // Legacy wallets only have one ScriptPubKeyMan which is a LegacyScriptPubKeyMan. + // Everything in m_internal_spk_managers and m_external_spk_managers point to the same legacyScriptPubKeyMan. + auto it = m_internal_spk_managers.find(OutputType::LEGACY); + if (it == m_internal_spk_managers.end()) return nullptr; + return dynamic_cast<LegacyScriptPubKeyMan*>(it->second); +} + +LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() +{ + SetupLegacyScriptPubKeyMan(); + return GetLegacyScriptPubKeyMan(); +} + +void CWallet::SetupLegacyScriptPubKeyMan() +{ + if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty()) { + return; + } + + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new LegacyScriptPubKeyMan(*this)); + for (const auto& type : OUTPUT_TYPES) { + m_internal_spk_managers[type] = spk_manager.get(); + m_external_spk_managers[type] = spk_manager.get(); + } + m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); } const CKeyingMaterial& CWallet::GetEncryptionKey() const @@ -4075,3 +4221,11 @@ bool CWallet::HasEncryptionKeys() const { return !mapMasterKeys.empty(); } + +void CWallet::ConnectScriptPubKeyManNotifiers() +{ + for (const auto& spk_man : GetActiveScriptPubKeyMans()) { + spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); + spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); + } +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c4511601de..a918bb8833 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,6 +35,8 @@ #include <boost/signals2/signal.hpp> +using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; + //! Explicitly unload and delete the wallet. //! Blocks the current thread after signaling the unload intent so that all //! wallet clients release the wallet. @@ -48,6 +50,7 @@ bool HasWallets(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::vector<std::string>& warnings); +std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet); enum class WalletCreationStatus { SUCCESS, @@ -310,6 +313,13 @@ public: enum AmountType { DEBIT, CREDIT, IMMATURE_CREDIT, AVAILABLE_CREDIT, AMOUNTTYPE_ENUM_ELEMENTS }; CAmount GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate = false) const; mutable CachableAmount m_amounts[AMOUNTTYPE_ENUM_ELEMENTS]; + /** + * This flag is true if all m_amounts caches are empty. This is particularly + * useful in places where MarkDirty is conditionally called and the + * condition can be expensive and thus can be skipped if the flag is true. + * See MarkDestinationsDirty. + */ + mutable bool m_is_cache_empty{true}; mutable bool fChangeCached; mutable bool fInMempool; mutable CAmount nChangeCached; @@ -436,6 +446,7 @@ public: m_amounts[IMMATURE_CREDIT].Reset(); m_amounts[AVAILABLE_CREDIT].Reset(); fChangeCached = false; + m_is_cache_empty = true; } void BindWallet(CWallet *pwalletIn) @@ -595,7 +606,7 @@ class WalletRescanReserver; //forward declarations for ScanForWalletTransactions class CWallet final : public WalletStorage, private interfaces::Chain::Notifications { private: - CKeyingMaterial vMasterKey GUARDED_BY(cs_KeyStore); + CKeyingMaterial vMasterKey GUARDED_BY(cs_wallet); bool Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no_keys = false); @@ -691,12 +702,19 @@ private: */ int m_last_block_processed_height GUARDED_BY(cs_wallet) = -1; + std::map<OutputType, ScriptPubKeyMan*> m_external_spk_managers; + std::map<OutputType, ScriptPubKeyMan*> m_internal_spk_managers; + + // Indexed by a unique identifier produced by each ScriptPubKeyMan using + // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure + std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + public: /* * Main wallet lock. * This lock protects all the fields added by CWallet. */ - mutable CCriticalSection cs_wallet; + mutable RecursiveMutex cs_wallet; /** Get database handle used by this wallet. Ideally this function would * not be necessary. @@ -798,10 +816,9 @@ public: bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - // Whether this or any UTXO with the same CTxDestination has been spent. - bool IsUsedDestination(const CTxDestination& dst) const; - bool IsUsedDestination(const uint256& hash, unsigned int n) const; - void SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + // Whether this or any known UTXO with the same single key has been spent. + bool IsUsedDestination(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SetUsedDestinationState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; @@ -913,9 +930,9 @@ public: * Should be called after CreateTransaction unless you want to abort * broadcasting the transaction. * - * @param tx[in] The transaction to be broadcast. - * @param mapValue[in] key-values to be set on the transaction. - * @param orderForm[in] BIP 70 / BIP 21 order form details to be set on the transaction. + * @param[in] tx The transaction to be broadcast. + * @param[in] mapValue key-values to be set on the transaction. + * @param[in] orderForm BIP 70 / BIP 21 order form details to be set on the transaction. */ void CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm); @@ -961,6 +978,12 @@ public: std::set<CTxDestination> GetLabelAddresses(const std::string& label) const; + /** + * Marks all outputs in each one of the destinations dirty, so their cache is + * reset and does not return outdated information. + */ + void MarkDestinationsDirty(const std::set<CTxDestination>& destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error); @@ -1116,28 +1139,34 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers + std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; + + //! Returns all unique ScriptPubKeyMans + std::set<ScriptPubKeyMan*> GetAllScriptPubKeyMans() const; + + //! Get the ScriptPubKeyMan for the given OutputType and internal/external chain. + ScriptPubKeyMan* GetScriptPubKeyMan(const OutputType& type, bool internal) const; + //! Get the ScriptPubKeyMan for a script ScriptPubKeyMan* GetScriptPubKeyMan(const CScript& script) const; + //! Get the ScriptPubKeyMan by id + ScriptPubKeyMan* GetScriptPubKeyMan(const uint256& id) const; //! Get the SigningProvider for a script - const SigningProvider* GetSigningProvider(const CScript& script) const; - const SigningProvider* GetSigningProvider(const CScript& script, SignatureData& sigdata) const; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script) const; + std::unique_ptr<SigningProvider> GetSigningProvider(const CScript& script, SignatureData& sigdata) const; + //! Get the LegacyScriptPubKeyMan which is used for all types, internal, and external. LegacyScriptPubKeyMan* GetLegacyScriptPubKeyMan() const; + LegacyScriptPubKeyMan* GetOrCreateLegacyScriptPubKeyMan(); + + //! Make a LegacyScriptPubKeyMan and set it for all types, internal, and external. + void SetupLegacyScriptPubKeyMan(); const CKeyingMaterial& GetEncryptionKey() const override; bool HasEncryptionKeys() const override; - // Temporary LegacyScriptPubKeyMan accessors and aliases. - friend class LegacyScriptPubKeyMan; - std::unique_ptr<LegacyScriptPubKeyMan> m_spk_man = MakeUnique<LegacyScriptPubKeyMan>(*this); - CCriticalSection& cs_KeyStore = m_spk_man->cs_KeyStore; - LegacyScriptPubKeyMan::KeyMap& mapKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapKeys; - LegacyScriptPubKeyMan::ScriptMap& mapScripts GUARDED_BY(cs_KeyStore) = m_spk_man->mapScripts; - LegacyScriptPubKeyMan::CryptedKeyMap& mapCryptedKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapCryptedKeys; - LegacyScriptPubKeyMan::WatchOnlySet& setWatchOnly GUARDED_BY(cs_KeyStore) = m_spk_man->setWatchOnly; - LegacyScriptPubKeyMan::WatchKeyMap& mapWatchKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapWatchKeys; - /** Get last block processed height */ int GetLastBlockHeight() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { @@ -1152,6 +1181,9 @@ public: m_last_block_processed_height = block_height; m_last_block_processed = block_hash; }; + + //! Connect the signals from ScriptPubKeyMans to the signals in CWallet + void ConnectScriptPubKeyManNotifiers(); }; /** diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 7d04b04764..a1928f45c4 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -196,7 +196,7 @@ public: static bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, - CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet, pwallet->GetLegacyScriptPubKeyMan()->cs_wallet) + CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { try { // Unserialize @@ -251,7 +251,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, char fYes; ssValue >> fYes; if (fYes == '1') { - pwallet->GetLegacyScriptPubKeyMan()->LoadWatchOnly(script); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadWatchOnly(script); } } else if (strType == DBKeys::KEY) { CPubKey vchPubKey; @@ -303,7 +303,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, strErr = "Error reading wallet database: CPrivKey corrupt"; return false; } - if (!pwallet->GetLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKey(key, vchPubKey)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadKey failed"; return false; @@ -334,7 +334,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssValue >> vchPrivKey; wss.nCKeys++; - if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCryptedKey(vchPubKey, vchPrivKey)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCryptedKey failed"; return false; @@ -346,14 +346,14 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->GetLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); } else if (strType == DBKeys::WATCHMETA) { CScript script; ssKey >> script; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - pwallet->GetLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadScriptMetadata(CScriptID(script), keyMeta); } else if (strType == DBKeys::DEFAULTKEY) { // We don't want or need the default key, but if there is one set, // we want to make sure that it is valid so that we can detect corruption @@ -369,13 +369,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CKeyPool keypool; ssValue >> keypool; - pwallet->GetLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadKeyPool(nIndex, keypool); } else if (strType == DBKeys::CSCRIPT) { uint160 hash; ssKey >> hash; CScript script; ssValue >> script; - if (!pwallet->GetLegacyScriptPubKeyMan()->LoadCScript(script)) + if (!pwallet->GetOrCreateLegacyScriptPubKeyMan()->LoadCScript(script)) { strErr = "Error reading wallet database: LegacyScriptPubKeyMan::LoadCScript failed"; return false; @@ -391,7 +391,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::HDCHAIN) { CHDChain chain; ssValue >> chain; - pwallet->GetLegacyScriptPubKeyMan()->SetHDChain(chain, true); + pwallet->GetOrCreateLegacyScriptPubKeyMan()->SetHDChain(chain, true); } else if (strType == DBKeys::FLAGS) { uint64_t flags; ssValue >> flags; @@ -434,7 +434,6 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) DBErrors result = DBErrors::LOAD_OK; LOCK(pwallet->cs_wallet); - AssertLockHeld(pwallet->GetLegacyScriptPubKeyMan()->cs_wallet); try { int nMinVersion = 0; if (m_batch.Read(DBKeys::MINVERSION, nMinVersion)) { @@ -516,8 +515,9 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) // nTimeFirstKey is only reliable if all keys have metadata if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { - auto spk_man = pwallet->GetLegacyScriptPubKeyMan(); + auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); if (spk_man) { + LOCK(spk_man->cs_KeyStore); spk_man->UpdateTimeFirstKey(1); } } @@ -713,7 +713,6 @@ bool WalletBatch::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, C { // Required in LoadKeyMetadata(): LOCK(dummyWallet->cs_wallet); - AssertLockHeld(dummyWallet->GetLegacyScriptPubKeyMan()->cs_wallet); fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, dummyWss, strType, strErr); } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index b1781d5ccf..1a65125480 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -124,7 +124,7 @@ public: std::string hdKeypath; //optional HD/bip32 keypath. Still used to determine whether a key is a seed. Also kept for backwards compatibility CKeyID hd_seed_id; //id of the HD seed used to derive this key KeyOriginInfo key_origin; // Key origin info with path and fingerprint - bool has_key_origin = false; //< Whether the key_origin is useful + bool has_key_origin = false; //!< Whether the key_origin is useful CKeyMetadata() { diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index dc0cac60bd..fbfdf9dd6b 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -27,6 +27,7 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: } // dummy chain interface std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), WalletDatabase::Create(path)), WalletToolReleaseWallet); + LOCK(wallet_instance->cs_wallet); bool first_run = true; DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); if (load_wallet_ret != DBErrors::LOAD_OK) { @@ -37,7 +38,7 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); // generate a new HD seed - auto spk_man = wallet_instance->GetLegacyScriptPubKeyMan(); + auto spk_man = wallet_instance->GetOrCreateLegacyScriptPubKeyMan(); CPubKey seed = spk_man->GenerateNewSeed(); spk_man->SetHDSeed(seed); diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index bd08da42d6..8ee3355f02 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 044c757e68..c91c9aca96 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/walletinitinterface.h b/src/walletinitinterface.h index 7ccda1c566..f4730273f1 100644 --- a/src/walletinitinterface.h +++ b/src/walletinitinterface.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/warnings.cpp b/src/warnings.cpp index 8df77096f1..467c3d0f65 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/warnings.h b/src/warnings.h index 58e5e4cd19..83b1add1ee 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp index a5f3be8f5b..aae760adde 100644 --- a/src/zmq/zmqabstractnotifier.cpp +++ b/src/zmq/zmqabstractnotifier.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h index e3fdbf7402..5f0036206d 100644 --- a/src/zmq/zmqconfig.h +++ b/src/zmq/zmqconfig.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index b3b97b6a2a..0ce14f232e 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 8bf9b0ba47..c820865497 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index ba89d1401d..04806903c2 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -112,7 +112,8 @@ bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) void CZMQAbstractPublishNotifier::Shutdown() { - assert(psocket); + // Early return if Initialize was not called + if (!psocket) return; int count = mapPublishNotifiers.count(address); diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index cf97b7ecce..5652877f3c 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index a70d9c4ac1..00f2833f55 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. """Combine logs from multiple bitcoin nodes as well as the test_framework log. This streams the combined log output to stdout. Use combine_logs.py > outputfile diff --git a/test/functional/create_cache.py b/test/functional/create_cache.py index 16a791177b..1108a8e354 100755 --- a/test/functional/create_cache.py +++ b/test/functional/create_cache.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Create a blockchain cache. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index fd69bbd2c7..99c88bbcc0 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 62c3eca07d..9b878e8bf8 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -29,7 +29,7 @@ class AbortNodeTest(BitcoinTestFramework): datadir = get_datadir_path(self.options.tmpdir, 0) # Deleting the undo file will result in reorg failure - os.unlink(os.path.join(datadir, 'regtest', 'blocks', 'rev00000.dat')) + os.unlink(os.path.join(datadir, self.chain, 'blocks', 'rev00000.dat')) # Connecting to a node with a more work chain will trigger a reorg # attempt. @@ -40,7 +40,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=60) + wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=200) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index c7e98bd4db..95905f477b 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1401,7 +1401,7 @@ class FullBlockTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() self.bootstrap_p2p(timeout=timeout) - def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=60): + def send_blocks(self, blocks, success=True, reject_reason=None, force_send=False, reconnect=False, timeout=960): """Sends blocks to test node. Syncs and verifies that tip has advanced to most recent block. Call with success = False if the tip shouldn't advance to the most recent block.""" diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index 6f01f97ea2..7bfad52c24 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the blocksdir option. diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 7d131e6045..2c6f2e733b 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -135,7 +135,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputs on {} failed with non-mandatory-script-verify-flag (Negative locktime)'.format(block.vtx[-1].hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Negative locktime)'.format(block.vtx[-1].hash)]): self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) self.nodes[0].p2p.sync_with_ping() diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 8397eb9f59..1a7c656274 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -39,7 +39,7 @@ class ConfArgsTest(BitcoinTestFramework): if self.is_wallet_compiled(): with open(inc_conf_file_path, 'w', encoding='utf8') as conf: conf.write("wallet=foo\n") - self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on regtest network when in [regtest] section.') + self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on %s network when in [%s] section.' % (self.chain, self.chain)) with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('regtest=0\n') # mainnet @@ -79,14 +79,44 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('') # clear def test_log_buffer(self): - with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0']): + with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0\n']): self.start_node(0, extra_args=['-noconnect=0']) self.stop_node(0) + def test_args_log(self): + self.log.info('Test config args logging') + with self.nodes[0].assert_debug_log( + expected_msgs=[ + 'Command-line arg: addnode="some.node"', + 'Command-line arg: rpcauth=****', + 'Command-line arg: rpcbind=****', + 'Command-line arg: rpcpassword=****', + 'Command-line arg: rpcuser=****', + 'Command-line arg: torpassword=****', + 'Config file arg: %s="1"' % self.chain, + 'Config file arg: [%s] server="1"' % self.chain, + ], + unexpected_msgs=[ + 'alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', + '127.1.1.1', + 'secret-rpcuser', + 'secret-torpassword', + ]): + self.start_node(0, extra_args=[ + '-addnode=some.node', + '-rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', + '-rpcbind=127.1.1.1', + '-rpcpassword=', + '-rpcuser=secret-rpcuser', + '-torpassword=secret-torpassword', + ]) + self.stop_node(0) + def run_test(self): self.stop_node(0) self.test_log_buffer() + self.test_args_log() self.test_config_file_parser() @@ -104,7 +134,7 @@ class ConfArgsTest(BitcoinTestFramework): # Check that using non-existent datadir in conf file fails conf_file = os.path.join(default_data_dir, "bitcoin.conf") - # datadir needs to be set before [regtest] section + # datadir needs to be set before [chain] section conf_file_contents = open(conf_file, encoding='utf8').read() with open(conf_file, 'w', encoding='utf8') as f: f.write("datadir=" + new_data_dir + "\n") @@ -116,17 +146,17 @@ class ConfArgsTest(BitcoinTestFramework): os.mkdir(new_data_dir) self.start_node(0, ['-conf='+conf_file, '-wallet=w1']) self.stop_node(0) - assert os.path.exists(os.path.join(new_data_dir, 'regtest', 'blocks')) + assert os.path.exists(os.path.join(new_data_dir, self.chain, 'blocks')) if self.is_wallet_compiled(): - assert os.path.exists(os.path.join(new_data_dir, 'regtest', 'wallets', 'w1')) + assert os.path.exists(os.path.join(new_data_dir, self.chain, 'wallets', 'w1')) # Ensure command line argument overrides datadir in conf os.mkdir(new_data_dir_2) self.nodes[0].datadir = new_data_dir_2 self.start_node(0, ['-datadir='+new_data_dir_2, '-conf='+conf_file, '-wallet=w2']) - assert os.path.exists(os.path.join(new_data_dir_2, 'regtest', 'blocks')) + assert os.path.exists(os.path.join(new_data_dir_2, self.chain, 'blocks')) if self.is_wallet_compiled(): - assert os.path.exists(os.path.join(new_data_dir_2, 'regtest', 'wallets', 'w2')) + assert os.path.exists(os.path.join(new_data_dir_2, self.chain, 'wallets', 'w2')) if __name__ == '__main__': diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 2ace96fef4..27da49cf24 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -120,7 +120,7 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputs on {} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)'.format(block.vtx[-1].hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)'.format(block.vtx[-1].hash)]): self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) self.nodes[0].p2p.sync_with_ping() diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index ba116e41f5..be1c044aa5 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" @@ -19,7 +19,7 @@ class FilelockTest(BitcoinTestFramework): self.nodes[0].wait_for_rpc_connection() def run_test(self): - datadir = os.path.join(self.nodes[0].datadir, 'regtest') + datadir = os.path.join(self.nodes[0].datadir, self.chain) self.log.info("Using datadir {}".format(datadir)) self.log.info("Check that we can't start a second bitcoind instance using the same datadir") diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index ed1d25c0d6..e3e2456183 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -17,7 +17,7 @@ class HelpTest(BitcoinTestFramework): # Don't start the node def get_node_output(self, *, ret_code_expected): - ret_code = self.nodes[0].process.wait(timeout=5) + ret_code = self.nodes[0].process.wait(timeout=60) assert_equal(ret_code, ret_code_expected) self.nodes[0].stdout.seek(0) self.nodes[0].stderr.seek(0) diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 2cd6a05d08..6f1a0cd348 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests the includeconf argument diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index bd4b271ca7..7f1e8f5bae 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -38,7 +38,7 @@ class LoadblockTest(BitcoinTestFramework): cfg_file = os.path.join(data_dir, "linearize.cfg") bootstrap_file = os.path.join(self.options.tmpdir, "bootstrap.dat") genesis_block = self.nodes[0].getblockhash(0) - blocks_dir = os.path.join(data_dir, "regtest", "blocks") + blocks_dir = os.path.join(data_dir, self.chain, "blocks") hash_list = tempfile.NamedTemporaryFile(dir=data_dir, mode='w', delete=False, diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index e6ff21ee9c..1bae29b302 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test debug logging.""" @@ -16,7 +16,7 @@ class LoggingTest(BitcoinTestFramework): self.setup_clean_chain = True def relative_log_path(self, name): - return os.path.join(self.nodes[0].datadir, "regtest", name) + return os.path.join(self.nodes[0].datadir, self.chain, name) def run_test(self): # test default log file name diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 42e2b5bff2..8d200915bd 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of -maxuploadtarget. diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index e1e0f00530..21b6b299f6 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -101,7 +101,7 @@ class PruneTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - self.prunedir = os.path.join(self.nodes[2].datadir, 'regtest', 'blocks', '') + self.prunedir = os.path.join(self.nodes[2].datadir, self.chain, 'blocks', '') connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) @@ -279,7 +279,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(ret, node.getblockchaininfo()['pruneheight']) def has_block(index): - return os.path.isfile(os.path.join(self.nodes[node_number].datadir, "regtest", "blocks", "blk{:05}.dat".format(index))) + return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", "blk{:05}.dat".format(index))) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py index 9485db9864..d782d3b1d8 100755 --- a/test/functional/feature_shutdown.py +++ b/test/functional/feature_shutdown.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind shutdown.""" diff --git a/test/functional/feature_uacomment.py b/test/functional/feature_uacomment.py index 85c250173f..4720f6dea3 100755 --- a/test/functional/feature_uacomment.py +++ b/test/functional/feature_uacomment.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -uacomment option.""" diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index e460db39f8..3b3b5bb0c1 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -33,7 +33,7 @@ class RPCInterfaceTest(BitcoinTestFramework): command = info['active_commands'][0] assert_equal(command['method'], 'getrpcinfo') assert_greater_than_or_equal(command['duration'], 0) - assert_equal(info['logpath'], os.path.join(self.nodes[0].datadir, 'regtest', 'debug.log')) + assert_equal(info['logpath'], os.path.join(self.nodes[0].datadir, self.chain, 'debug.log')) def test_batch_request(self): self.log.info("Testing basic JSON-RPC batch request...") diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 5aea10fbce..89c55f31f3 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -59,6 +59,10 @@ class ZMQTest (BitcoinTestFramework): # Note that the publishing order is not defined in the documentation and # is subject to change. import zmq + + # Invalid zmq arguments don't take down the node, see #17185. + self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) + address = 'tcp://127.0.0.1:28332' socket = self.ctx.socket(zmq.SUB) socket.set(zmq.RCVTIMEO, 60000) diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 9cb6b03ee0..900cabccda 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -8,6 +8,7 @@ from io import BytesIO import math from test_framework.test_framework import BitcoinTestFramework +from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, @@ -20,6 +21,9 @@ from test_framework.script import ( hash160, CScript, OP_0, + OP_2, + OP_3, + OP_CHECKMULTISIG, OP_EQUAL, OP_HASH160, OP_RETURN, @@ -35,7 +39,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [[ - '-txindex', + '-txindex','-permitbaremultisig=0', ]] * self.num_nodes self.supports_cli = False @@ -262,6 +266,15 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() + tx.vout[0].scriptPubKey = CScript([OP_2, pubkey, pubkey, pubkey, OP_3, OP_CHECKMULTISIG]) # Some bare multisig script (2-of-3) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bare-multisig'}], + rawtxs=[tx.serialize().hex()], + ) + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'scriptsig-not-pushonly'}], diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 398fdeb326..d3690b245e 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. @@ -117,8 +117,8 @@ class MempoolPersistTest(BitcoinTestFramework): wait_until(lambda: self.nodes[0].getmempoolinfo()["loaded"]) assert_equal(len(self.nodes[0].getrawmempool()), 5) - mempooldat0 = os.path.join(self.nodes[0].datadir, 'regtest', 'mempool.dat') - mempooldat1 = os.path.join(self.nodes[1].datadir, 'regtest', 'mempool.dat') + mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') + mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) self.nodes[0].savemempool() diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 123f0b4c28..3b148d5cf0 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -76,7 +76,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_101_id = self.nodes[0].sendrawtransaction(spend_101_raw) spend_102_1_id = self.nodes[0].sendrawtransaction(spend_102_1_raw) - self.sync_all() + self.sync_all(timeout=360) assert_equal(set(self.nodes[0].getrawmempool()), {spend_101_id, spend_102_1_id, timelock_tx_id}) @@ -91,7 +91,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): for node in self.nodes: node.invalidateblock(new_blocks[0]) - self.sync_all() + self.sync_all(timeout=360) # mempool should be empty. assert_equal(set(self.nodes[0].getrawmempool()), set()) diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index bfeaa76c74..8262e30592 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -70,7 +70,7 @@ class MiningTest(BitcoinTestFramework): self.log.info('getmininginfo') mining_info = node.getmininginfo() assert_equal(mining_info['blocks'], 200) - assert_equal(mining_info['chain'], 'regtest') + assert_equal(mining_info['chain'], self.chain) assert 'currentblocktx' not in mining_info assert 'currentblockweight' not in mining_info assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10')) diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 1b64b65b82..9047fc6828 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 20864881c1..9876d749ff 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -78,7 +78,7 @@ class InvalidMessagesTest(BitcoinTestFramework): # Peer 1, despite serving up a bunch of nonsense, should still be connected. self.log.info("Waiting for node to drop junk messages.") - node.p2p.sync_with_ping(timeout=320) + node.p2p.sync_with_ping(timeout=400) assert node.p2p.is_connected # @@ -145,13 +145,13 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_magic_bytes(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - def swap_magic_bytes(): + async def swap_magic_bytes(): conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes conn.magic_bytes = b'\x00\x11\x22\x32' # Call .result() to block until the atomic swap is complete, otherwise # we might run into races later on - asyncio.run_coroutine_threadsafe(asyncio.coroutine(swap_magic_bytes)(), NetworkThread.network_event_loop).result() + asyncio.run_coroutine_threadsafe(swap_magic_bytes(), NetworkThread.network_event_loop).result() with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): conn.send_message(messages.msg_ping(nonce=0xff)) diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 3cca2d78db..5975a52b2a 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid transactions. diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 40b28d7533..93e2957fd0 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p permission message. @@ -7,21 +7,35 @@ Test that permissions are correctly calculated and applied """ +from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.messages import ( + CTransaction, + CTxInWitness, + FromHex, +) +from test_framework.mininode import P2PDataStore +from test_framework.script import ( + CScript, + OP_TRUE, +) from test_framework.test_node import ErrorMatch from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, p2p_port, + wait_until, ) + class P2PPermissionsTests(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[],[]] def run_test(self): + self.check_tx_relay() + self.checkpermission( # default permissions (no specific permissions) ["-whitelist=127.0.0.1"], @@ -48,9 +62,9 @@ class P2PPermissionsTests(BitcoinTestFramework): ip_port = "127.0.0.1:{}".format(p2p_port(1)) self.replaceinconfig(1, "bind=127.0.0.1", "whitebind=bloomfilter,forcerelay@" + ip_port) self.checkpermission( - ["-whitelist=noban@127.0.0.1" ], + ["-whitelist=noban@127.0.0.1"], # Check parameter interaction forcerelay should activate relay - ["noban", "bloomfilter", "forcerelay", "relay" ], + ["noban", "bloomfilter", "forcerelay", "relay"], False) self.replaceinconfig(1, "whitebind=bloomfilter,forcerelay@" + ip_port, "bind=127.0.0.1") @@ -83,6 +97,41 @@ class P2PPermissionsTests(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(["-whitelist=noban@127.0.0.1:230"], "Invalid netmask specified in", match=ErrorMatch.PARTIAL_REGEX) self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1/10"], "Cannot resolve -whitebind address", match=ErrorMatch.PARTIAL_REGEX) + def check_tx_relay(self): + block_op_true = self.nodes[0].getblock(self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) + self.sync_all() + + self.log.debug("Create a connection from a whitelisted wallet that rebroadcasts raw txs") + # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only + # rebroadcast via the inv-getdata mechanism. However, even for whitelisted connections, a full node would + # currently not request a txid that is already in the mempool. + self.restart_node(1, extra_args=["-whitelist=forcerelay@127.0.0.1"]) + p2p_rebroadcast_wallet = self.nodes[1].add_p2p_connection(P2PDataStore()) + + self.log.debug("Send a tx from the wallet initially") + tx = FromHex( + CTransaction(), + self.nodes[0].createrawtransaction( + inputs=[{ + 'txid': block_op_true['tx'][0], + 'vout': 0, + }], outputs=[{ + ADDRESS_BCRT1_P2WSH_OP_TRUE: 5, + }]), + ) + tx.wit.vtxinwit = [CTxInWitness()] + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + txid = tx.rehash() + + self.log.debug("Wait until tx is in node[1]'s mempool") + p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) + + self.log.debug("Check that node[1] will send the tx to node[0] even though it is already in the mempool") + connect_nodes(self.nodes[1], 0) + with self.nodes[1].assert_debug_log(["Force relaying tx {} from whitelisted peer=0".format(txid)]): + p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) + wait_until(lambda: txid in self.nodes[0].getrawmempool()) + def checkpermission(self, args, expectedPermissions, whitelisted): self.restart_node(1, args) connect_nodes(self.nodes[0], 1) @@ -95,9 +144,10 @@ class P2PPermissionsTests(BitcoinTestFramework): def replaceinconfig(self, nodeid, old, new): with open(self.nodes[nodeid].bitcoinconf, encoding="utf8") as f: - newText=f.read().replace(old, new) + newText = f.read().replace(old, new) with open(self.nodes[nodeid].bitcoinconf, 'w', encoding="utf8") as f: f.write(newText) + if __name__ == '__main__': P2PPermissionsTests().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 297fa88fbe..3223c27e0b 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -31,7 +31,7 @@ from test_framework.messages import ( msg_inv, msg_tx, msg_block, - msg_witness_tx, + msg_no_witness_tx, ser_uint256, ser_vector, sha256, @@ -125,10 +125,11 @@ def test_transaction_acceptance(node, p2p, tx, with_witness, accepted, reason=No - use the getrawmempool rpc to check for acceptance.""" reason = [reason] if reason else [] with node.assert_debug_log(expected_msgs=reason): - p2p.send_message(msg_witness_tx(tx) if with_witness else msg_tx(tx)) + p2p.send_message(msg_tx(tx) if with_witness else msg_no_witness_tx(tx)) p2p.sync_with_ping() assert_equal(tx.hash in node.getrawmempool(), accepted) + def test_witness_block(node, p2p, block, accepted, with_witness=True, reason=None): """Send a block to the node and check that it's accepted @@ -311,9 +312,9 @@ class SegWitTest(BitcoinTestFramework): # Check that serializing it with or without witness is the same # This is a sanity check of our testing framework. - assert_equal(msg_tx(tx).serialize(), msg_witness_tx(tx).serialize()) + assert_equal(msg_no_witness_tx(tx).serialize(), msg_tx(tx).serialize()) - self.test_node.send_message(msg_witness_tx(tx)) + self.test_node.send_message(msg_tx(tx)) self.test_node.sync_with_ping() # make sure the tx was processed assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 7527bdfb08..438e7f68e0 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -25,7 +25,7 @@ class DumptxoutsetTest(BitcoinTestFramework): FILENAME = 'txoutset.dat' out = node.dumptxoutset(FILENAME) - expected_path = Path(node.datadir) / 'regtest' / FILENAME + expected_path = Path(node.datadir) / self.chain / FILENAME assert expected_path.is_file() diff --git a/test/functional/rpc_getaddressinfo_label_deprecation.py b/test/functional/rpc_getaddressinfo_label_deprecation.py new file mode 100755 index 0000000000..5e739ebede --- /dev/null +++ b/test/functional/rpc_getaddressinfo_label_deprecation.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test deprecation of the RPC getaddressinfo `label` field. It has been +superceded by the `labels` field. + +""" +from test_framework.test_framework import BitcoinTestFramework + +class GetAddressInfoLabelDeprecationTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = False + # Start node[0] with -deprecatedrpc=label, and node[1] without. + self.extra_args = [["-deprecatedrpc=label"], []] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_label_with_deprecatedrpc_flag(self): + self.log.info("Test getaddressinfo label with -deprecatedrpc flag") + node = self.nodes[0] + address = node.getnewaddress() + info = node.getaddressinfo(address) + assert "label" in info + + def test_label_without_deprecatedrpc_flag(self): + self.log.info("Test getaddressinfo label without -deprecatedrpc flag") + node = self.nodes[1] + address = node.getnewaddress() + info = node.getaddressinfo(address) + assert "label" not in info + + def run_test(self): + """Test getaddressinfo label with and without -deprecatedrpc flag.""" + self.test_label_with_deprecatedrpc_flag() + self.test_label_without_deprecatedrpc_flag() + + +if __name__ == '__main__': + GetAddressInfoLabelDeprecationTest().main() diff --git a/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py new file mode 100755 index 0000000000..3f2e8dee18 --- /dev/null +++ b/test/functional/rpc_getaddressinfo_labels_purpose_deprecation.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test deprecation of RPC getaddressinfo `labels` returning an array +containing a JSON object of `name` and purpose` key-value pairs. It now +returns an array containing only the label name. + +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +LABELS_TO_TEST = frozenset({"" , "New 𝅘𝅥𝅯 $<#>&!рыба Label"}) + +class GetAddressInfoLabelsPurposeDeprecationTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = False + # Start node[0] with -deprecatedrpc=labelspurpose and node[1] without. + self.extra_args = [["-deprecatedrpc=labelspurpose"], []] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_labels(self, node_num, label_name, expected_value): + node = self.nodes[node_num] + address = node.getnewaddress() + if label_name != "": + node.setlabel(address, label_name) + self.log.info(" set label to {}".format(label_name)) + labels = node.getaddressinfo(address)["labels"] + self.log.info(" labels = {}".format(labels)) + assert_equal(labels, expected_value) + + def run_test(self): + """Test getaddressinfo labels with and without -deprecatedrpc flag.""" + self.log.info("Test getaddressinfo labels with -deprecatedrpc flag") + for label in LABELS_TO_TEST: + self.test_labels(node_num=0, label_name=label, expected_value=[{"name": label, "purpose": "receive"}]) + + self.log.info("Test getaddressinfo labels without -deprecatedrpc flag") + for label in LABELS_TO_TEST: + self.test_labels(node_num=1, label_name=label, expected_value=[label]) + + +if __name__ == '__main__': + GetAddressInfoLabelsPurposeDeprecationTest().main() diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index ec8ea5697a..027ae368e7 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC help output.""" diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 368debf35c..376bb35f07 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC calls related to net. diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 2cc9650cb2..33af819d34 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -422,5 +422,20 @@ class PSBTTest(BitcoinTestFramework): assert_equal(analysis['next'], 'creator') assert_equal(analysis['error'], 'PSBT is not valid. Input 0 spends unspendable output') + self.log.info("PSBT with invalid values should have error message and Creator as next") + analysis = self.nodes[0].analyzepsbt('cHNidP8BAHECAAAAAfA00BFgAm6tp86RowwH6BMImQNL5zXUcTT97XoLGz0BAAAAAAD/////AgD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XL87QKVAAAAABYAFPck4gF7iL4NL4wtfRAKgQbghiTUAAAAAAABAR8AgIFq49AHABYAFJUDtxf2PHo641HEOBOAIvFMNTr2AAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 has invalid value') + + analysis = self.nodes[0].analyzepsbt('cHNidP8BAHECAAAAAfA00BFgAm6tp86RowwH6BMImQNL5zXUcTT97XoLGz0BAAAAAAD/////AgCAgWrj0AcAFgAUKNw0x8HRctAgmvoevm4u1SbN7XL87QKVAAAAABYAFPck4gF7iL4NL4wtfRAKgQbghiTUAAAAAAABAR8A8gUqAQAAABYAFJUDtxf2PHo641HEOBOAIvFMNTr2AAAA') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Output amount invalid') + + analysis = self.nodes[0].analyzepsbt('cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + assert_equal(analysis['next'], 'creator') + assert_equal(analysis['error'], 'PSBT is not valid. Input 0 specifies invalid prevout') + + assert_raises_rpc_error(-25, 'Missing inputs', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index f31e4f43bd..c3d34be0dd 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -54,7 +54,7 @@ class ScantxoutsetTest(BitcoinTestFramework): self.log.info("Stop node, remove wallet, mine again some blocks...") self.stop_node(0) - shutil.rmtree(os.path.join(self.nodes[0].datadir, "regtest", 'wallets')) + shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')) self.start_node(0) self.nodes[0].generate(110) diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index b1d2b6f431..1cc1fb164b 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -26,7 +26,7 @@ class SetBanTests(BitcoinTestFramework): self.nodes[1].setban("127.0.0.1", "add") # Node 0 should not be able to reconnect - with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'], timeout=5): + with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'], timeout=50): self.restart_node(1, []) self.nodes[0].addnode("127.0.0.1:" + str(p2p_port(1)), "onetry") diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index 8913b8698d..ca8be42d3b 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" diff --git a/test/functional/rpc_users.py b/test/functional/rpc_users.py index 3541f96ab7..b75ce15f2e 100755 --- a/test/functional/rpc_users.py +++ b/test/functional/rpc_users.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiple RPC users.""" diff --git a/test/functional/rpc_whitelist.py b/test/functional/rpc_whitelist.py index 219132410b..219132410b 100644..100755 --- a/test/functional/rpc_whitelist.py +++ b/test/functional/rpc_whitelist.py diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 97585fe054..6a7e91216a 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -13,6 +13,8 @@ from . import segwit_addr ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj' ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97' +# Coins sent to this address can be spent with a witness stack of just OP_TRUE +ADDRESS_BCRT1_P2WSH_OP_TRUE = 'bcrt1qft5p2uhsdcdc3l2ua4ap5qqfg4pjaqlp250x7us7a8qqhrxrxfsqseac85' class AddressType(enum.Enum): diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 25520a2151..4f7a9a8b13 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1105,17 +1105,17 @@ class msg_tx: self.tx.deserialize(f) def serialize(self): - return self.tx.serialize_without_witness() + return self.tx.serialize_with_witness() def __repr__(self): return "msg_tx(tx=%s)" % (repr(self.tx)) -class msg_witness_tx(msg_tx): +class msg_no_witness_tx(msg_tx): __slots__ = () def serialize(self): - return self.tx.serialize_with_witness() + return self.tx.serialize_without_witness() class msg_block: diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e5c77ae5fa..0742dbe617 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -298,7 +298,9 @@ class TestNode(): wait_until(self.is_node_stopped, timeout=timeout) @contextlib.contextmanager - def assert_debug_log(self, expected_msgs, timeout=2): + def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): + if unexpected_msgs is None: + unexpected_msgs = [] time_end = time.time() + timeout debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: @@ -313,6 +315,9 @@ class TestNode(): dl.seek(prev_size) log = dl.read() print_log = " - " + "\n - ".join(log.splitlines()) + for unexpected_msg in unexpected_msgs: + if re.search(re.escape(unexpected_msg), log, flags=re.MULTILINE): + self._raise_assertion_error('Unexpected message "{}" partially matches log:\n\n{}\n\n'.format(unexpected_msg, print_log)) for expected_msg in expected_msgs: if re.search(re.escape(expected_msg), log, flags=re.MULTILINE) is None: found = False diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index 3d81a61120..eb537015fb 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful util functions for testing the wallet""" @@ -88,11 +88,6 @@ def get_multisig(node): p2sh_p2wsh_script=CScript([OP_HASH160, witness_script, OP_EQUAL]).hex(), p2sh_p2wsh_addr=script_to_p2sh_p2wsh(script_code)) -def labels_value(name="", purpose="receive"): - """Generate a getaddressinfo labels array from a name and purpose. - Often used as the value of a labels kwarg for calling test_address below.""" - return [{"name": name, "purpose": purpose}] - def test_address(node, address, **kwargs): """Get address info for `address` and test whether the returned values are as expected.""" addr_info = node.getaddressinfo(address) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 110733c529..8b527bcff0 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -212,6 +212,8 @@ BASE_SCRIPTS = [ 'p2p_permissions.py', 'feature_blocksdir.py', 'feature_config_args.py', + 'rpc_getaddressinfo_labels_purpose_deprecation.py', + 'rpc_getaddressinfo_label_deprecation.py', 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 32ef257456..d2629ff1ed 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -27,7 +27,7 @@ class ToolWalletTest(BitcoinTestFramework): def bitcoin_wallet_process(self, *args): binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] - args = ['-datadir={}'.format(self.nodes[0].datadir), '-regtest'] + list(args) + args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] + list(args) return subprocess.Popen([binary] + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) def assert_raises_tool_error(self, error, *args): @@ -198,7 +198,7 @@ class ToolWalletTest(BitcoinTestFramework): self.log.debug('Wallet file shasum unchanged\n') def run_test(self): - self.wallet_path = os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat') + self.wallet_path = os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat') self.test_invalid_tool_commands_and_args() # Warning: The following tests are order-dependent. self.test_tool_wallet_info() diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 16925ea753..1ca02a695c 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the avoid_reuse and setwalletflag features.""" @@ -86,7 +86,13 @@ class AvoidReuseTest(BitcoinTestFramework): reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) self.test_fund_send_fund_senddirty() reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) - self.test_fund_send_fund_send() + self.test_fund_send_fund_send("legacy") + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_fund_send_fund_send("p2sh-segwit") + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_fund_send_fund_send("bech32") + reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) + self.test_getbalances_used() def test_persistence(self): '''Test that wallet files persist the avoid_reuse flag.''' @@ -182,7 +188,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_approx(self.nodes[1].getbalance(), 5, 0.001) assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 5, 0.001) - def test_fund_send_fund_send(self): + def test_fund_send_fund_send(self, second_addr_type): ''' Test the simple case where [1] generates a new address A, then [0] sends 10 BTC to A. @@ -193,7 +199,7 @@ class AvoidReuseTest(BitcoinTestFramework): ''' self.log.info("Test fund send fund send") - fundaddr = self.nodes[1].getnewaddress() + fundaddr = self.nodes[1].getnewaddress(label="", address_type="legacy") retaddr = self.nodes[0].getnewaddress() self.nodes[0].sendtoaddress(fundaddr, 10) @@ -214,7 +220,19 @@ class AvoidReuseTest(BitcoinTestFramework): # getbalances should show no used, 5 btc trusted assert_balances(self.nodes[1], mine={"used": 0, "trusted": 5}) - self.nodes[0].sendtoaddress(fundaddr, 10) + # For the second send, we transmute it to a related single-key address + # to make sure it's also detected as re-use + fund_spk = self.nodes[0].getaddressinfo(fundaddr)["scriptPubKey"] + fund_decoded = self.nodes[0].decodescript(fund_spk) + if second_addr_type == "p2sh-segwit": + new_fundaddr = fund_decoded["segwit"]["p2sh-segwit"] + elif second_addr_type == "bech32": + new_fundaddr = fund_decoded["segwit"]["addresses"][0] + else: + new_fundaddr = fundaddr + assert_equal(second_addr_type, "legacy") + + self.nodes[0].sendtoaddress(new_fundaddr, 10) self.nodes[0].generate(1) self.sync_all() @@ -240,5 +258,35 @@ class AvoidReuseTest(BitcoinTestFramework): assert_approx(self.nodes[1].getbalance(), 1, 0.001) assert_approx(self.nodes[1].getbalance(avoid_reuse=False), 11, 0.001) + def test_getbalances_used(self): + ''' + getbalances and listunspent should pick up on reused addresses + immediately, even for address reusing outputs created before the first + transaction was spending from that address + ''' + self.log.info("Test getbalances used category") + + # node under test should be completely empty + assert_equal(self.nodes[1].getbalance(avoid_reuse=False), 0) + + new_addr = self.nodes[1].getnewaddress() + ret_addr = self.nodes[0].getnewaddress() + + # send multiple transactions, reusing one address + for _ in range(11): + self.nodes[0].sendtoaddress(new_addr, 1) + + self.nodes[0].generate(1) + self.sync_all() + + # send transaction that should not use all the available outputs + # per the current coin selection algorithm + self.nodes[1].sendtoaddress(ret_addr, 5) + + # getbalances and listunspent should show the remaining outputs + # in the reused address as used/reused + assert_unspent(self.nodes[1], total_count=2, total_sum=6, reused_count=1, reused_sum=1) + assert_balances(self.nodes[1], mine={"used": 1, "trusted": 5}) + if __name__ == '__main__': AvoidReuseTest().main() diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index bb835dc811..16c69f304a 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -107,9 +107,9 @@ class WalletBackupTest(BitcoinTestFramework): self.stop_node(2) def erase_three(self): - os.remove(os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat')) - os.remove(os.path.join(self.nodes[1].datadir, 'regtest', 'wallets', 'wallet.dat')) - os.remove(os.path.join(self.nodes[2].datadir, 'regtest', 'wallets', 'wallet.dat')) + os.remove(os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat')) + os.remove(os.path.join(self.nodes[1].datadir, self.chain, 'wallets', 'wallet.dat')) + os.remove(os.path.join(self.nodes[2].datadir, self.chain, 'wallets', 'wallet.dat')) def run_test(self): self.log.info("Generating initial blockchain") @@ -167,13 +167,13 @@ class WalletBackupTest(BitcoinTestFramework): self.erase_three() # Start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'chainstate')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) # Restore wallets from backup - shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat')) - shutil.copyfile(os.path.join(self.nodes[1].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, 'regtest', 'wallets', 'wallet.dat')) - shutil.copyfile(os.path.join(self.nodes[2].datadir, 'wallet.bak'), os.path.join(self.nodes[2].datadir, 'regtest', 'wallets', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[1].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[2].datadir, 'wallet.bak'), os.path.join(self.nodes[2].datadir, self.chain, 'wallets', 'wallet.dat')) self.log.info("Re-starting nodes") self.start_three() @@ -188,8 +188,8 @@ class WalletBackupTest(BitcoinTestFramework): self.erase_three() #start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, 'regtest', 'chainstate')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) + shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) self.start_three() @@ -209,10 +209,10 @@ class WalletBackupTest(BitcoinTestFramework): # Backup to source wallet file must fail sourcePaths = [ - os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'wallet.dat'), - os.path.join(self.nodes[0].datadir, 'regtest', '.', 'wallets', 'wallet.dat'), - os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', ''), - os.path.join(self.nodes[0].datadir, 'regtest', 'wallets')] + os.path.join(self.nodes[0].datadir, self.chain, 'wallets', 'wallet.dat'), + os.path.join(self.nodes[0].datadir, self.chain, '.', 'wallets', 'wallet.dat'), + os.path.join(self.nodes[0].datadir, self.chain, 'wallets', ''), + os.path.join(self.nodes[0].datadir, self.chain, 'wallets')] for sourcePath in sourcePaths: assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 2606afa0e4..15746d312c 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -15,10 +15,7 @@ from test_framework.util import ( connect_nodes, wait_until, ) -from test_framework.wallet_util import ( - labels_value, - test_address, -) +from test_framework.wallet_util import test_address class WalletTest(BitcoinTestFramework): @@ -395,7 +392,7 @@ class WalletTest(BitcoinTestFramework): for label in [u'рыба', u'𝅘𝅥𝅯']: addr = self.nodes[0].getnewaddress() self.nodes[0].setlabel(addr, label) - test_address(self.nodes[0], addr, label=label, labels=labels_value(name=label)) + test_address(self.nodes[0], addr, labels=[label]) assert label in self.nodes[0].listlabels() self.nodes[0].rpc.ensure_ascii = True # restore to default diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 0c08655833..4eb0d19a4f 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -78,6 +78,7 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_fails(rbf_node, dest_address) test_dust_to_fee(rbf_node, dest_address) test_settxfee(rbf_node, dest_address) + test_watchonly_psbt(self, peer_node, rbf_node, dest_address) test_rebumping(rbf_node, dest_address) test_rebumping_not_replaceable(rbf_node, dest_address) test_unconfirmed_not_spendable(rbf_node, rbf_node_address) @@ -103,6 +104,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] > -rbftx["fee"] assert_equal(bumped_tx["origfee"], -rbftx["fee"]) + assert "psbt" not in bumped_tx # check that bumped_tx propagates, original tx was evicted and has a wallet conflict self.sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() @@ -280,6 +282,86 @@ def test_maxtxfee_fails(test, rbf_node, dest_address): test.restart_node(1, test.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) +def test_watchonly_psbt(test, peer_node, rbf_node, dest_address): + priv_rec_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0/*)#rweraev0" + pub_rec_desc = rbf_node.getdescriptorinfo(priv_rec_desc)["descriptor"] + priv_change_desc = "wpkh([00000001/84'/1'/0']tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/*)#j6uzqvuh" + pub_change_desc = rbf_node.getdescriptorinfo(priv_change_desc)["descriptor"] + # Create a wallet with private keys that can sign PSBTs + rbf_node.createwallet(wallet_name="signer", disable_private_keys=False, blank=True) + signer = rbf_node.get_wallet_rpc("signer") + assert signer.getwalletinfo()['private_keys_enabled'] + result = signer.importmulti([{ + "desc": priv_rec_desc, + "timestamp": 0, + "range": [0,1], + "internal": False, + "keypool": False # Keys can only be imported to the keypool when private keys are disabled + }, + { + "desc": priv_change_desc, + "timestamp": 0, + "range": [0, 0], + "internal": True, + "keypool": False + }]) + assert_equal(result, [{'success': True}, {'success': True}]) + + # Create another wallet with just the public keys, which creates PSBTs + rbf_node.createwallet(wallet_name="watcher", disable_private_keys=True, blank=True) + watcher = rbf_node.get_wallet_rpc("watcher") + assert not watcher.getwalletinfo()['private_keys_enabled'] + + result = watcher.importmulti([{ + "desc": pub_rec_desc, + "timestamp": 0, + "range": [0,10], + "internal": False, + "keypool": True, + "watchonly": True + }, + { + "desc": pub_change_desc, + "timestamp": 0, + "range": [0, 10], + "internal": True, + "keypool": True, + "watchonly": True + }]) + assert_equal(result, [{'success': True}, {'success': True}]) + + funding_address1 = watcher.getnewaddress(address_type='bech32') + funding_address2 = watcher.getnewaddress(address_type='bech32') + peer_node.sendmany("", {funding_address1: 0.001, funding_address2: 0.001}) + peer_node.generate(1) + test.sync_all() + + # Create single-input PSBT for transaction to be bumped + psbt = watcher.walletcreatefundedpsbt([], {dest_address:0.0005}, 0, {"feeRate": 0.00001}, True)['psbt'] + psbt_signed = signer.walletprocesspsbt(psbt=psbt, sign=True, sighashtype="ALL", bip32derivs=True) + psbt_final = watcher.finalizepsbt(psbt_signed["psbt"]) + original_txid = watcher.sendrawtransaction(psbt_final["hex"]) + assert_equal(len(watcher.decodepsbt(psbt)["tx"]["vin"]), 1) + + # Bump fee, obnoxiously high to add additional watchonly input + bumped_psbt = watcher.bumpfee(original_txid, {"fee_rate":0.005}) + assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) + assert "txid" not in bumped_psbt + assert_equal(bumped_psbt["origfee"], -watcher.gettransaction(original_txid)["fee"]) + assert not watcher.finalizepsbt(bumped_psbt["psbt"])["complete"] + + # Sign bumped transaction + bumped_psbt_signed = signer.walletprocesspsbt(psbt=bumped_psbt["psbt"], sign=True, sighashtype="ALL", bip32derivs=True) + bumped_psbt_final = watcher.finalizepsbt(bumped_psbt_signed["psbt"]) + assert bumped_psbt_final["complete"] + + # Broadcast bumped transaction + bumped_txid = watcher.sendrawtransaction(bumped_psbt_final["hex"]) + assert bumped_txid in rbf_node.getrawmempool() + assert original_txid not in rbf_node.getrawmempool() + + rbf_node.unloadwallet("watcher") + rbf_node.unloadwallet("signer") def test_rebumping(rbf_node, dest_address): # check that re-bumping the original tx fails, but bumping the bumper succeeds diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 53edf710b9..a39dfc7895 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -137,7 +137,7 @@ class WalletDumpTest(BitcoinTestFramework): # encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') - self.nodes[0].walletpassphrase('test', 10) + self.nodes[0].walletpassphrase('test', 100) # Should be a no-op: self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(wallet_enc_dump) diff --git a/test/functional/wallet_encryption.py b/test/functional/wallet_encryption.py index fbcb4e75ba..bc7e3cca59 100755 --- a/test/functional/wallet_encryption.py +++ b/test/functional/wallet_encryption.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Wallet encryption""" diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index d1178611bd..f2fa1d3e40 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet group functionality.""" @@ -16,7 +16,7 @@ class WalletGroupTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [[], [], ['-avoidpartialspends']] - self.rpc_timeout = 120 + self.rpc_timeout = 240 def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 575b501f50..7497475b67 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -68,11 +68,11 @@ class WalletHDTest(BitcoinTestFramework): self.log.info("Restore backup ...") self.stop_node(1) - # we need to delete the complete regtest directory + # we need to delete the complete chain directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks")) - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallets", "wallet.dat")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) + shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) self.start_node(1) # Assert that derivation is deterministic @@ -93,9 +93,9 @@ class WalletHDTest(BitcoinTestFramework): # Try a RPC based rescan self.stop_node(1) - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "blocks")) - shutil.rmtree(os.path.join(self.nodes[1].datadir, "regtest", "chainstate")) - shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, "regtest", "wallets", "wallet.dat")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "blocks")) + shutil.rmtree(os.path.join(self.nodes[1].datadir, self.chain, "chainstate")) + shutil.copyfile(os.path.join(self.nodes[1].datadir, "hd.bak"), os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat")) self.start_node(1, extra_args=self.extra_args[1]) connect_nodes(self.nodes[0], 1) self.sync_all() diff --git a/test/functional/wallet_import_with_label.py b/test/functional/wallet_import_with_label.py index e356fce469..6a9d2e8290 100755 --- a/test/functional/wallet_import_with_label.py +++ b/test/functional/wallet_import_with_label.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the behavior of RPC importprivkey on set and unset labels of @@ -11,10 +11,7 @@ with and without a label. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.wallet_util import ( - labels_value, - test_address, -) +from test_framework.wallet_util import test_address class ImportWithLabel(BitcoinTestFramework): @@ -39,8 +36,7 @@ class ImportWithLabel(BitcoinTestFramework): address, iswatchonly=True, ismine=False, - label=label, - labels=labels_value(name=label)) + labels=[label]) self.log.info( "Import the watch-only address's private key without a " @@ -48,11 +44,7 @@ class ImportWithLabel(BitcoinTestFramework): ) priv_key = self.nodes[0].dumpprivkey(address) self.nodes[1].importprivkey(priv_key) - - test_address(self.nodes[1], - address, - label=label, - labels=labels_value(name=label)) + test_address(self.nodes[1], address, labels=[label]) self.log.info( "Test importaddress without label and importprivkey with label." @@ -64,8 +56,7 @@ class ImportWithLabel(BitcoinTestFramework): address2, iswatchonly=True, ismine=False, - label="", - labels=labels_value()) + labels=[""]) self.log.info( "Import the watch-only address's private key with a " @@ -75,10 +66,7 @@ class ImportWithLabel(BitcoinTestFramework): label2 = "Test Label 2" self.nodes[1].importprivkey(priv_key2, label2) - test_address(self.nodes[1], - address2, - label=label2, - labels=labels_value(name=label2)) + test_address(self.nodes[1], address2, labels=[label2]) self.log.info("Test importaddress with label and importprivkey with label.") self.log.info("Import a watch-only address with a label.") @@ -89,8 +77,7 @@ class ImportWithLabel(BitcoinTestFramework): address3, iswatchonly=True, ismine=False, - label=label3_addr, - labels=labels_value(name=label3_addr)) + labels=[label3_addr]) self.log.info( "Import the watch-only address's private key with a " @@ -100,10 +87,7 @@ class ImportWithLabel(BitcoinTestFramework): label3_priv = "Test Label 3 for importprivkey" self.nodes[1].importprivkey(priv_key3, label3_priv) - test_address(self.nodes[1], - address3, - label=label3_priv, - labels=labels_value(name=label3_priv)) + test_address(self.nodes[1], address3, labels=[label3_priv]) self.log.info( "Test importprivkey won't label new dests with the same " @@ -117,8 +101,7 @@ class ImportWithLabel(BitcoinTestFramework): address4, iswatchonly=True, ismine=False, - label=label4_addr, - labels=labels_value(name=label4_addr), + labels=[label4_addr], embedded=None) self.log.info( @@ -131,15 +114,9 @@ class ImportWithLabel(BitcoinTestFramework): self.nodes[1].importprivkey(priv_key4) embedded_addr = self.nodes[1].getaddressinfo(address4)['embedded']['address'] - test_address(self.nodes[1], - embedded_addr, - label="", - labels=labels_value()) + test_address(self.nodes[1], embedded_addr, labels=[""]) - test_address(self.nodes[1], - address4, - label=label4_addr, - labels=labels_value(name=label4_addr)) + test_address(self.nodes[1], address4, labels=[label4_addr]) self.stop_nodes() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 5febac5998..f152fcd1a4 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -29,7 +29,6 @@ from test_framework.util import ( from test_framework.wallet_util import ( get_key, get_multisig, - labels_value, test_address, ) @@ -570,8 +569,7 @@ class ImportMultiTest(BitcoinTestFramework): key.p2sh_p2wpkh_addr, solvable=True, ismine=True, - label=p2sh_p2wpkh_label, - labels=labels_value(name=p2sh_p2wpkh_label)) + labels=[p2sh_p2wpkh_label]) # Test ranged descriptor fails if range is not specified xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" @@ -642,8 +640,7 @@ class ImportMultiTest(BitcoinTestFramework): key.p2pkh_addr, solvable=True, ismine=False, - label=p2pkh_label, - labels=labels_value(name=p2pkh_label)) + labels=[p2pkh_label]) # Test import fails if both desc and scriptPubKey are provided key = get_key(self.nodes[0]) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 2e70a9e0a5..829633a050 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -30,7 +30,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - wallet_path = os.path.join(self.nodes[1].datadir, "regtest", "wallets", "wallet.dat") + wallet_path = os.path.join(self.nodes[1].datadir, self.chain, "wallets", "wallet.dat") wallet_backup_path = os.path.join(self.nodes[1].datadir, "wallet.bak") self.nodes[0].generate(101) diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index 27371d43bb..337d2e55d9 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test label RPCs. @@ -13,10 +13,8 @@ from collections import defaultdict from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error -from test_framework.wallet_util import ( - labels_value, - test_address, -) +from test_framework.wallet_util import test_address + class WalletLabelsTest(BitcoinTestFramework): def set_test_params(self): @@ -157,12 +155,7 @@ class Label: if self.receive_address is not None: assert self.receive_address in self.addresses for address in self.addresses: - test_address( - node, - address, - label=self.name, - labels=labels_value(name=self.name, purpose=self.purpose[address]) - ) + test_address(node, address, labels=[self.name]) assert self.name in node.listlabels() assert_equal( node.getaddressesbylabel(self.name), diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index afd473306d..b0590b149a 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -11,10 +11,7 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.wallet_util import ( - labels_value, - test_address, -) +from test_framework.wallet_util import test_address class ReceivedByTest(BitcoinTestFramework): @@ -131,7 +128,7 @@ class ReceivedByTest(BitcoinTestFramework): # set pre-state label = '' address = self.nodes[1].getnewaddress() - test_address(self.nodes[1], address, label=label, labels=labels_value(name=label)) + test_address(self.nodes[1], address, labels=[label]) received_by_label_json = [r for r in self.nodes[1].listreceivedbylabel() if r["label"] == label][0] balance_by_label = self.nodes[1].getreceivedbylabel(label) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 85d900f6cc..f2fa41b647 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -38,7 +38,7 @@ class MultiWalletTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] - data_dir = lambda *p: os.path.join(node.datadir, 'regtest', *p) + data_dir = lambda *p: os.path.join(node.datadir, self.chain, *p) wallet_dir = lambda *p: data_dir('wallets', *p) wallet = lambda name: node.get_wallet_rpc(name) @@ -186,7 +186,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_equal(w4.getbalance(), 3) batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) - assert_equal(batch[0]["result"]["chain"], "regtest") + assert_equal(batch[0]["result"]["chain"], self.chain) assert_equal(batch[1]["result"]["walletname"], "w1") self.log.info('Check for per-wallet settxfee call') diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index f48018e9fb..497a5dd95e 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -90,7 +90,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): # Node0 wallet file is loaded on longest sync'ed node1 self.stop_node(1) self.nodes[0].backupwallet(os.path.join(self.nodes[0].datadir, 'wallet.bak')) - shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, 'regtest', 'wallet.dat')) + shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, self.chain, 'wallet.dat')) self.start_node(1) tx_after_reorg = self.nodes[1].gettransaction(txid) # Check that normal confirmed tx is confirmed again but with different blockhash diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index bbdd047465..50e86cf9dc 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -15,11 +15,13 @@ import logging # Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus FUZZERS_MISSING_CORPORA = [ "addr_info_deserialize", + "asmap", "base_encode_decode", "block", "block_file_info_deserialize", "block_filter_deserialize", "block_header_and_short_txids_deserialize", + "decode_tx", "fee_rate_deserialize", "flat_file_pos_deserialize", "hex", @@ -38,6 +40,7 @@ FUZZERS_MISSING_CORPORA = [ "psbt_output_deserialize", "pub_key_deserialize", "script_deserialize", + "strprintf", "sub_net_deserialize", "tx_in", "tx_in_deserialize", diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py index a33ab17f3f..0a4cc875d0 100755 --- a/test/lint/check-rpc-mappings.py +++ b/test/lint/check-rpc-mappings.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check RPC argument consistency.""" diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 5603456e62..ff3f784437 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 7b5707a17a..caa7affc63 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2015 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh index a4c6f0a8d4..1aacc09bcc 100755 --- a/test/lint/lint-assertions.sh +++ b/test/lint/lint-assertions.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 1538ad77b1..bbd94dd6c7 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -13,11 +13,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "index/txindex -> validation -> index/txindex" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/bantablemodel -> qt/clientmodel -> qt/bantablemodel" - "qt/bitcoingui -> qt/utilitydialog -> qt/bitcoingui" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" - "qt/bitcoingui -> qt/walletview -> qt/bitcoingui" - "qt/clientmodel -> qt/peertablemodel -> qt/clientmodel" "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" @@ -25,7 +21,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" "policy/fees -> txmempool -> validation -> policy/fees" - "wallet/scriptpubkeyman -> wallet/wallet -> wallet/scriptpubkeyman" ) EXIT_CODE=0 diff --git a/test/lint/lint-filenames.sh b/test/lint/lint-filenames.sh index 6716cac0fe..3f7491cd2b 100755 --- a/test/lint/lint-filenames.sh +++ b/test/lint/lint-filenames.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index 99127e01f8..2870432bff 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -17,12 +17,13 @@ FALSE_POSITIVES = [ ("src/index/base.cpp", "FatalError(const char* fmt, const Args&... args)"), ("src/netbase.cpp", "LogConnectFailure(bool manual_connection, const char* fmt, const Args&... args)"), ("src/util/system.cpp", "strprintf(_(COPYRIGHT_HOLDERS).translated, COPYRIGHT_HOLDERS_SUBSTITUTION)"), + ("src/validationinterface.cpp", "LogPrint(BCLog::VALIDATION, fmt \"\\n\", __VA_ARGS__)"), ("src/wallet/wallet.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), ("src/wallet/wallet.h", "LogPrintf((\"%s \" + fmt).c_str(), GetDisplayName(), parameters...)"), + ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(std::string fmt, Params... parameters)"), + ("src/wallet/scriptpubkeyman.h", "LogPrintf((\"%s \" + fmt).c_str(), m_storage.GetDisplayName(), parameters...)"), ("src/logging.h", "LogPrintf(const char* fmt, const Args&... args)"), ("src/wallet/scriptpubkeyman.h", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), - ("src/wallet/scriptpubkeyman.cpp", "WalletLogPrintf(fmt, parameters...)"), - ("src/wallet/scriptpubkeyman.cpp", "WalletLogPrintf(const std::string& fmt, const Params&... parameters)"), ] diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index cb630c78ad..184c3682c8 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -34,7 +34,7 @@ if ! python3 -m doctest test/lint/lint-format-strings.py; then fi for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue)"); do + for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do MATCHING_FILES+=("${MATCHING_FILE}") done if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index 0d654e796e..2d3beaf582 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index d27e45a23f..bb2bd4e56c 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index ea9ea7a58d..35e58c2df6 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -1,17 +1,35 @@ #!/usr/bin/env bash +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C KNOWN_VIOLATIONS=( "src/bitcoin-tx.cpp.*stoul" + "src/bitcoin-tx.cpp.*std::to_string" "src/bitcoin-tx.cpp.*trim_right" "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" + "src/qt/optionsmodel.cpp.*std::to_string" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" + "src/rpc/net.cpp.*std::to_string" + "src/rpc/rawtransaction.cpp.*std::to_string" + "src/rpc/util.cpp.*std::to_string" + "src/test/addrman_tests.cpp.*std::to_string" + "src/test/blockchain_tests.cpp.*std::to_string" "src/test/dbwrapper_tests.cpp:.*snprintf" + "src/test/denialofservice_tests.cpp.*std::to_string" "src/test/fuzz/parse_numbers.cpp:.*atoi" + "src/test/key_tests.cpp.*std::to_string" + "src/test/net_tests.cpp.*std::to_string" + "src/test/settings_tests.cpp.*std::to_string" + "src/test/timedata_tests.cpp.*std::to_string" + "src/test/util/setup_common.cpp.*std::to_string" + "src/test/util_tests.cpp.*std::to_string" + "src/test/util_threadnames_tests.cpp.*std::to_string" "src/torcontrol.cpp:.*atoi" "src/torcontrol.cpp:.*strtol" "src/util/strencodings.cpp:.*atoi" @@ -19,6 +37,7 @@ KNOWN_VIOLATIONS=( "src/util/strencodings.cpp:.*strtoul" "src/util/strencodings.h:.*atoi" "src/util/system.cpp:.*atoi" + "src/wallet/scriptpubkeyman.cpp.*std::to_string" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" @@ -93,6 +112,7 @@ LOCALE_DEPENDENT_FUNCTIONS=( snprintf sprintf sscanf + std::to_string stod stof stoi diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh index 72b6b2e9d5..2fbb4a38e7 100755 --- a/test/lint/lint-logs.sh +++ b/test/lint/lint-logs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 2b1a1d8fbc..86ac5a930f 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-shebang.sh b/test/lint/lint-shebang.sh index fda22592d3..a666fdfecf 100755 --- a/test/lint/lint-shebang.sh +++ b/test/lint/lint-shebang.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + # Assert expected shebang lines export LC_ALL=C diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index 251a94f7ff..c7a3d0de44 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 70eea34363..b9c5c038d0 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -7,6 +7,14 @@ deadlock:WalletBatch # Intentional deadlock in tests deadlock:TestPotentialDeadLockDetected +# Race due to unprotected calls to thread-unsafe BOOST_TEST_MESSAGE from different threads: +# * G_TEST_LOG_FUN in the index thread +# * boost test case invoker (entering a test case) in the main thread +# TODO: get rid of BOOST_ macros, see also https://github.com/bitcoin/bitcoin/issues/8670 +race:blockfilter_index_initial_sync_invoker +race:txindex_initial_sync_invoker +race:validation_block_tests::TestSubscriber + # Wildcard for all gui tests, should be replaced with non-wildcard suppressions race:src/qt/test/* deadlock:src/qt/test/* |