diff options
73 files changed, 2314 insertions, 458 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 6dc029ee51..e353101ecc 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -143,10 +143,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [focal]' + name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:hirsute env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" diff --git a/build_msvc/README.md b/build_msvc/README.md index 5ce0f6cde4..88a05644a7 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,7 +3,7 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2017 and 2019. +Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md). @@ -35,7 +35,7 @@ vcpkg integrate install Qt --------------------- -In order to build the Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. +In order to build Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v142) and platform type (x86 or x64) must also match. Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. @@ -57,19 +57,13 @@ PS >py -3 msvc-autogen.py - An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. -- To build from the command line with the Visual Studio 2017 toolchain use: - -``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build -``` - - To build from the command line with the Visual Studio 2019 toolchain use: ``` msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build ``` -- Alternatively open the `build_msvc/bitcoin.sln` file in Visual Studio. +- Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. AppVeyor --------------------- @@ -81,3 +75,25 @@ For safety reasons the Bitcoin Core .appveyor.yml file has the artifact options #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip ``` + +Security +--------------------- +[Base address randomization](https://docs.microsoft.com/en-us/cpp/build/reference/dynamicbase-use-address-space-layout-randomization?view=msvc-160) is used to make Bitcoin Core more secure. When building Bitcoin using the `build_msvc` process base address randomization can be disabled by editing `common.init.vcproj` to change `RandomizedBaseAddress` from `true` to `false` and then rebuilding the project. + +To check if `bitcoind` has `RandomizedBaseAddress` enabled or disabled run + +``` +.\dumpbin.exe /headers src/bitcoind.exe +``` + +If is it enabled then in the output `Dynamic base` will be listed in the `DLL characteristics` under `OPTIONAL HEADER VALUES` as shown below + +``` + 8160 DLL characteristics + High Entropy Virtual Addresses + Dynamic base + NX compatible + Terminal Server Aware +``` + +This may not disable all stack randomization as versions of windows employ additional stack randomization protections. These protections must be turned off in the OS configuration.
\ No newline at end of file diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index f195d3d451..6ea018d846 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -105,6 +105,7 @@ <Link> <SubSystem>Console</SubSystem> <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> + <RandomizedBaseAddress>true</RandomizedBaseAddress> </Link> <Lib> <AdditionalOptions>/ignore:4221</AdditionalOptions> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 490ce8b1ce..96bb584375 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -34,6 +34,7 @@ <ClCompile Include="..\..\src\qt\overviewpage.cpp" /> <ClCompile Include="..\..\src\qt\paymentserver.cpp" /> <ClCompile Include="..\..\src\qt\peertablemodel.cpp" /> + <ClCompile Include="..\..\src\qt\peertablesortproxy.cpp" /> <ClCompile Include="..\..\src\qt\platformstyle.cpp" /> <ClCompile Include="..\..\src\qt\psbtoperationsdialog.cpp" /> <ClCompile Include="..\..\src\qt\qrimagewidget.cpp" /> @@ -87,6 +88,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_overviewpage.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_paymentserver.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablemodel.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablesortproxy.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_platformstyle.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_psbtoperationsdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qrimagewidget.cpp" /> diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 6039c51018..92af98aa9b 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:20.04 +export DOCKER_NAME_TAG=ubuntu:hirsute export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++ --enable-external-signer" diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 2d9a4a2153..db99260eb9 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -56,7 +56,7 @@ Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple Apple uses `clang` extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of `-F`, -`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when +`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when building for macOS. Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the diff --git a/depends/README.md b/depends/README.md index 6b20791281..50e1a32c70 100644 --- a/depends/README.md +++ b/depends/README.md @@ -49,6 +49,11 @@ The paths are automatically configured and no other options are needed unless ta sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libz-dev python3-setuptools libtinfo5 xorriso +Note: You must obtain the macOS SDK before proceeding with a cross-compile. +Under the depends directory, create a subdirectory named `SDKs`. +Then, place the extracted SDK under this new directory. +For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction). + #### For Win64 cross compilation - see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index f4103fc1f2..001c928424 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -1,5 +1,5 @@ -build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path) -build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path) +build_darwin_CC:=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path) +build_darwin_CXX:=$(shell xcrun -f clang++) -isysroot$(shell xcrun --show-sdk-path) build_darwin_AR:=$(shell xcrun -f ar) build_darwin_RANLIB:=$(shell xcrun -f ranlib) build_darwin_STRIP:=$(shell xcrun -f strip) @@ -10,8 +10,8 @@ build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o #darwin host on darwin builder. overrides darwin host preferences. -darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(shell xcrun --show-sdk-path) -darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ --sysroot $(shell xcrun --show-sdk-path) +darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path) +darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path) darwin_AR:=$(shell xcrun -f ar) darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index dd71697f0f..4e9187c9e0 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -100,7 +100,7 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u LIBRARY_PATH \ $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - --sysroot=$(OSX_SDK) \ + -isysroot$(OSX_SDK) \ -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ @@ -108,7 +108,7 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u LIBRARY_PATH \ $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ - --sysroot=$(OSX_SDK) \ + -isysroot$(OSX_SDK) \ -stdlib=libc++ -nostdinc++ \ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \ -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 5fe027fb8a..783f87ca7c 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,8 +1,8 @@ package=native_mac_alias -$(package)_version=2.1.1 +$(package)_version=2.2.0 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz -$(package)_sha256_hash=c0ffceee14f7d04a6eb323fb7b8217dc3f373b346198d2ca42300a8362db7efa +$(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages define $(package)_build_cmds diff --git a/doc/files.md b/doc/files.md index a80adf30ea..695ea6f448 100644 --- a/doc/files.md +++ b/doc/files.md @@ -53,6 +53,7 @@ Subdirectory | File(s) | Description `indexes/txindex/` | LevelDB database | Transaction index; *optional*, used if `-txindex=1` `indexes/blockfilter/basic/db/` | LevelDB database | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic` `indexes/blockfilter/basic/` | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic` +`indexes/coinstats/db/` | LevelDB database | Coinstats index; *optional*, used if `-coinstatsindex=1` `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, wallets reside in the [data directory](#data-directory-location) `./` | `anchors.dat` | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup `./` | `banlist.dat` | Stores the IPs/subnets of banned nodes diff --git a/doc/release-notes/release-notes-0.21.1.md b/doc/release-notes/release-notes-0.21.1.md new file mode 100644 index 0000000000..d032fa8429 --- /dev/null +++ b/doc/release-notes/release-notes-0.21.1.md @@ -0,0 +1,203 @@ +0.21.1 Release Notes +==================== + +Bitcoin Core version 0.21.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.21.1/> + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Notable changes +=============== + +## Taproot Soft Fork + +Included in this release are the mainnet and testnet activation +parameters for the taproot soft fork (BIP341) which also adds support +for schnorr signatures (BIP340) and tapscript (BIP342). + +If activated, these improvements will allow users of single-signature +scripts, multisignature scripts, and complex contracts to all use +identical-appearing commitments that enhance their privacy and the +fungibility of all bitcoins. Spenders will enjoy lower fees and the +ability to resolve many multisig scripts and complex contracts with the +same efficiency, low fees, and large anonymity set as single-sig users. +Taproot and schnorr also include efficiency improvements for full nodes +such as the ability to batch signature verification. Together, the +improvements lay the groundwork for future potential +upgrades that may improve efficiency, privacy, and fungibility further. + +Activation for taproot is being managed using a variation of BIP9 +versionbits called Speedy Trial (described in BIP341). Taproot's +versionbit is bit 2, and nodes will begin tracking which blocks signal +support for taproot at the beginning of the first retarget period after +taproot’s start date of 24 April 2021. If 90% of blocks within a +2,016-block retarget period (about two weeks) signal support for taproot +prior to the first retarget period beginning after the time of 11 August +2021, the soft fork will be locked in, and taproot will then be active +as of block 709632 (expected in early or mid November). + +Should taproot not be locked in via Speedy Trial activation, it is +expected that a follow-up activation mechanism will be deployed, with +changes to address the reasons the Speedy Trial method failed. + +This release includes the ability to pay taproot addresses, although +payments to such addresses are not secure until taproot activates. +It also includes the ability to relay and mine taproot transactions +after activation. Beyond those two basic capabilities, this release +does not include any code that allows anyone to directly use taproot. +The addition of taproot-related features to Bitcoin Core's wallet is +expected in later releases once taproot activation is assured. + +All users, businesses, and miners are encouraged to upgrade to this +release (or a subsequent compatible release) unless they object to +activation of taproot. If taproot is locked in, then upgrading before +block 709632 is highly recommended to help enforce taproot's new rules +and to avoid the unlikely case of seeing falsely confirmed transactions. + +Miners who want to activate Taproot should preferably use this release +to control their signaling. The `getblocktemplate` RPC results will +automatically be updated to signal once the appropriate start has been +reached and continue signaling until the timeout occurs or taproot +activates. Alternatively, miners may manually start signaling on bit 2 +at any time; if taproot activates, they will need to ensure they update +their nodes before block 709632 or non-upgraded nodes could cause them to mine on +an invalid chain. See the [versionbits +FAQ](https://bitcoincore.org/en/2016/06/08/version-bits-miners-faq/) for +details. + + +For more information about taproot, please see the following resources: + +- Technical specifications + - [BIP340 Schnorr signatures for secp256k1](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki) + - [BIP341 Taproot: SegWit version 1 spending rules](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki) + - [BIP342 Validation of Taproot scripts](https://github.com/bitcoin/bips/blob/master/bip-0342.mediawiki) + +- Popular articles; + - [Taproot Is Coming: What It Is, and How It Will Benefit Bitcoin](https://bitcoinmagazine.com/technical/taproot-coming-what-it-and-how-it-will-benefit-bitcoin) + - [What do Schnorr Signatures Mean for Bitcoin?](https://academy.binance.com/en/articles/what-do-schnorr-signatures-mean-for-bitcoin) + - [The Schnorr Signature & Taproot Softfork Proposal](https://blog.bitmex.com/the-schnorr-signature-taproot-softfork-proposal/) + +- Development history overview + - [Taproot](https://bitcoinops.org/en/topics/taproot/) + - [Schnorr signatures](https://bitcoinops.org/en/topics/schnorr-signatures/) + - [Tapscript](https://bitcoinops.org/en/topics/tapscript/) + - [Soft fork activation](https://bitcoinops.org/en/topics/soft-fork-activation/) + +- Other + - [Questions and answers related to taproot](https://bitcoin.stackexchange.com/questions/tagged/taproot) + - [Taproot review](https://github.com/ajtowns/taproot-review) + +Updated RPCs +------------ + +- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) + being implemented, behavior for all RPCs that accept addresses is changed when + a native witness version 1 (or higher) is passed. These now require a Bech32m + encoding instead of a Bech32 one, and Bech32m encoding will be used for such + addresses in RPC output as well. No version 1 addresses should be created + for mainnet until consensus rules are adopted that give them meaning + (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). + Once that happens, Bech32m is expected to be used for them, so this shouldn't + affect any production systems, but may be observed on other networks where such + addresses already have meaning (like signet). + +0.21.1 change log +================= + +### Consensus +- #21377 Speedy trial support for versionbits (ajtowns) +- #21686 Speedy trial activation parameters for Taproot (achow101) + +### P2P protocol and network code +- #20852 allow CSubNet of non-IP networks (vasild) +- #21043 Avoid UBSan warning in ProcessMessage(…) (practicalswift) + +### Wallet +- #21166 Introduce DeferredSignatureChecker and have SignatureExtractorClass subclass it (achow101) +- #21083 Avoid requesting fee rates multiple times during coin selection (achow101) + +### RPC and other APIs +- #21201 Disallow sendtoaddress and sendmany when private keys disabled (achow101) + +### Build system +- #21486 link against -lsocket if required for `*ifaddrs` (fanquake) +- #20983 Fix MSVC build after gui#176 (hebasto) + +### Tests and QA +- #21380 Add fuzzing harness for versionbits (ajtowns) +- #20812 fuzz: Bump FuzzedDataProvider.h (MarcoFalke) +- #20740 fuzz: Update FuzzedDataProvider.h from upstream (LLVM) (practicalswift) +- #21446 Update vcpkg checkout commit (sipsorcery) +- #21397 fuzz: Bump FuzzedDataProvider.h (MarcoFalke) +- #21081 Fix the unreachable code at `feature_taproot` (brunoerg) +- #20562 Test that a fully signed tx given to signrawtx is unchanged (achow101) +- #21571 Make sure non-IP peers get discouraged and disconnected (vasild, MarcoFalke) +- #21489 fuzz: cleanups for versionbits fuzzer (ajtowns) + +### Miscellaneous +- #20861 BIP 350: Implement Bech32m and use it for v1+ segwit addresses (sipa) + +### Documentation +- #21384 add signet to bitcoin.conf documentation (jonatack) +- #21342 Remove outdated comment (hebasto) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Andrew Chow +- Anthony Towns +- Bruno Garcia +- Fabian Jahr +- fanquake +- Hennadii Stepanov +- Jon Atack +- Luke Dashjr +- MarcoFalke +- Pieter Wuille +- practicalswift +- randymcmillan +- Sjors Provoost +- Vasil Dimov +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 5b7fc776a4..85c67c8470 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -142,8 +142,11 @@ # both prior transactions and several dozen future transactions. #keypool=100 +# Maintain coinstats index used by the gettxoutsetinfo RPC (default: 0). +#coinstatsindex=1 + # Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex and -rescan. +# This mode is incompatible with -txindex, -coinstatsindex and -rescan. # 0 = default (no pruning). # 1 = allows manual pruning via RPC. # >=550 = target to stay under in MiB. diff --git a/src/Makefile.am b/src/Makefile.am index 447015fc66..972a3e279b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -153,6 +153,7 @@ BITCOIN_CORE_H = \ i2p.h \ index/base.h \ index/blockfilterindex.h \ + index/coinstatsindex.h \ index/disktxpos.h \ index/txindex.h \ indirectmap.h \ @@ -326,6 +327,7 @@ libbitcoin_server_a_SOURCES = \ i2p.cpp \ index/base.cpp \ index/blockfilterindex.cpp \ + index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ mapport.cpp \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index a573247afa..caa8500ffa 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -61,6 +61,7 @@ QT_MOC_CPP = \ qt/moc_optionsmodel.cpp \ qt/moc_overviewpage.cpp \ qt/moc_peertablemodel.cpp \ + qt/moc_peertablesortproxy.cpp \ qt/moc_paymentserver.cpp \ qt/moc_psbtoperationsdialog.cpp \ qt/moc_qrimagewidget.cpp \ @@ -134,6 +135,7 @@ BITCOIN_QT_H = \ qt/overviewpage.h \ qt/paymentserver.h \ qt/peertablemodel.h \ + qt/peertablesortproxy.h \ qt/platformstyle.h \ qt/psbtoperationsdialog.h \ qt/qrimagewidget.h \ @@ -232,6 +234,7 @@ BITCOIN_QT_BASE_CPP = \ qt/optionsdialog.cpp \ qt/optionsmodel.cpp \ qt/peertablemodel.cpp \ + qt/peertablesortproxy.cpp \ qt/platformstyle.cpp \ qt/qvalidatedlineedit.cpp \ qt/qvaluecombobox.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 570f011f7a..9360aa05f1 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -80,6 +80,7 @@ BITCOIN_TESTS =\ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ test/coins_tests.cpp \ + test/coinstatsindex_tests.cpp \ test/compilerbug_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ @@ -274,6 +275,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/random.cpp \ test/fuzz/rbf.cpp \ test/fuzz/rolling_bloom_filter.cpp \ + test/fuzz/rpc.cpp \ test/fuzz/script.cpp \ test/fuzz/script_assets_test_minimizer.cpp \ test/fuzz/script_bitcoin_consensus.cpp \ diff --git a/src/compressor.cpp b/src/compressor.cpp index a70306d320..ef3135e7a5 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -52,7 +52,7 @@ static bool IsToPubKey(const CScript& script, CPubKey &pubkey) return false; } -bool CompressScript(const CScript& script, std::vector<unsigned char> &out) +bool CompressScript(const CScript& script, CompressedScript& out) { CKeyID keyID; if (IsToKeyID(script, keyID)) { @@ -92,7 +92,7 @@ unsigned int GetSpecialScriptSize(unsigned int nSize) return 0; } -bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &in) +bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScript& in) { switch(nSize) { case 0x00: diff --git a/src/compressor.h b/src/compressor.h index 478bfff0b6..40b2496f06 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -6,14 +6,26 @@ #ifndef BITCOIN_COMPRESSOR_H #define BITCOIN_COMPRESSOR_H +#include <prevector.h> #include <primitives/transaction.h> #include <script/script.h> #include <serialize.h> #include <span.h> -bool CompressScript(const CScript& script, std::vector<unsigned char> &out); +/** + * This saves us from making many heap allocations when serializing + * and deserializing compressed scripts. + * + * This prevector size is determined by the largest .resize() in the + * CompressScript function. The largest compressed script format is a + * compressed public key, which is 33 bytes. + */ +using CompressedScript = prevector<33, unsigned char>; + + +bool CompressScript(const CScript& script, CompressedScript& out); unsigned int GetSpecialScriptSize(unsigned int nSize); -bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out); +bool DecompressScript(CScript& script, unsigned int nSize, const CompressedScript& in); /** * Compress amount. @@ -51,7 +63,7 @@ struct ScriptCompression template<typename Stream> void Ser(Stream &s, const CScript& script) { - std::vector<unsigned char> compr; + CompressedScript compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); return; @@ -66,7 +78,7 @@ struct ScriptCompression unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { - std::vector<unsigned char> vch(GetSpecialScriptSize(nSize), 0x00); + CompressedScript vch(GetSpecialScriptSize(nSize), 0x00); s >> MakeSpan(vch); DecompressScript(script, nSize, vch); return; diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index e5a0d4cb9c..a2b769cd56 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -341,6 +341,6 @@ MuHash3072& MuHash3072::Insert(Span<const unsigned char> in) noexcept { } MuHash3072& MuHash3072::Remove(Span<const unsigned char> in) noexcept { - m_numerator.Divide(ToNum3072(in)); + m_denominator.Multiply(ToNum3072(in)); return *this; } diff --git a/src/index/base.h b/src/index/base.h index ac3c429a5a..d887620524 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -81,6 +81,8 @@ protected: void ChainStateFlushed(const CBlockLocator& locator) override; + const CBlockIndex* CurrentIndex() { return m_best_block_index.load(); }; + /// Initialize internal state from the database and block index. virtual bool Init(); diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp new file mode 100644 index 0000000000..c7c1f4b533 --- /dev/null +++ b/src/index/coinstatsindex.cpp @@ -0,0 +1,472 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <coins.h> +#include <crypto/muhash.h> +#include <index/coinstatsindex.h> +#include <node/blockstorage.h> +#include <serialize.h> +#include <txdb.h> +#include <undo.h> +#include <validation.h> + +static constexpr char DB_BLOCK_HASH = 's'; +static constexpr char DB_BLOCK_HEIGHT = 't'; +static constexpr char DB_MUHASH = 'M'; + +namespace { + +struct DBVal { + uint256 muhash; + uint64_t transaction_output_count; + uint64_t bogo_size; + CAmount total_amount; + CAmount total_subsidy; + CAmount block_unspendable_amount; + CAmount block_prevout_spent_amount; + CAmount block_new_outputs_ex_coinbase_amount; + CAmount block_coinbase_amount; + CAmount unspendables_genesis_block; + CAmount unspendables_bip30; + CAmount unspendables_scripts; + CAmount unspendables_unclaimed_rewards; + + SERIALIZE_METHODS(DBVal, obj) + { + READWRITE(obj.muhash); + READWRITE(obj.transaction_output_count); + READWRITE(obj.bogo_size); + READWRITE(obj.total_amount); + READWRITE(obj.total_subsidy); + READWRITE(obj.block_unspendable_amount); + READWRITE(obj.block_prevout_spent_amount); + READWRITE(obj.block_new_outputs_ex_coinbase_amount); + READWRITE(obj.block_coinbase_amount); + READWRITE(obj.unspendables_genesis_block); + READWRITE(obj.unspendables_bip30); + READWRITE(obj.unspendables_scripts); + READWRITE(obj.unspendables_unclaimed_rewards); + } +}; + +struct DBHeightKey { + int height; + + explicit DBHeightKey(int height_in) : height(height_in) {} + + template <typename Stream> + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template <typename Stream> + void Unserialize(Stream& s) + { + char prefix{static_cast<char>(ser_readdata8(s))}; + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for coinstatsindex DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 block_hash; + + explicit DBHashKey(const uint256& hash_in) : block_hash(hash_in) {} + + SERIALIZE_METHODS(DBHashKey, obj) + { + char prefix{DB_BLOCK_HASH}; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for coinstatsindex DB hash key"); + } + + READWRITE(obj.block_hash); + } +}; + +}; // namespace + +std::unique_ptr<CoinStatsIndex> g_coin_stats_index; + +CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) +{ + fs::path path{GetDataDir() / "indexes" / "coinstats"}; + fs::create_directories(path); + + m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); +} + +bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy += block_subsidy; + + // Ignore genesis block + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair<uint256, DBVal> read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + if (read_out.first != expected_block_hash) { + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + + // TODO: Deduplicate BIP30 related code + bool is_bip30_block{(pindex->nHeight == 91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight == 91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; + + // Add the new utxos created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx{block.vtx.at(i)}; + + // Skip duplicate txid coinbase transactions (BIP30). + if (is_bip30_block && tx->IsCoinBase()) { + m_block_unspendable_amount += block_subsidy; + m_unspendables_bip30 += block_subsidy; + continue; + } + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out{tx->vout[j]}; + Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)}; + + // Skip unspendable coins + if (coin.out.scriptPubKey.IsUnspendable()) { + m_block_unspendable_amount += coin.out.nValue; + m_unspendables_scripts += coin.out.nValue; + continue; + } + + m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + if (tx->IsCoinBase()) { + m_block_coinbase_amount += coin.out.nValue; + } else { + m_block_new_outputs_ex_coinbase_amount += coin.out.nValue; + } + + ++m_transaction_output_count; + m_total_amount += coin.out.nValue; + m_bogo_size += GetBogoSize(coin.out.scriptPubKey); + } + + // The coinbase tx has no undo data since no former output is spent + if (!tx->IsCoinBase()) { + const auto& tx_undo{block_undo.vtxundo.at(i - 1)}; + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin{tx_undo.vprevout[j]}; + COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; + + m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + m_block_prevout_spent_amount += coin.out.nValue; + + --m_transaction_output_count; + m_total_amount -= coin.out.nValue; + m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); + } + } + } + } else { + // genesis block + m_block_unspendable_amount += block_subsidy; + m_unspendables_genesis_block += block_subsidy; + } + + // If spent prevouts + block subsidy are still a higher amount than + // new outputs + coinbase + current unspendable amount this means + // the miner did not claim the full block reward. Unclaimed block + // rewards are also unspendable. + const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)}; + m_block_unspendable_amount += unclaimed_rewards; + m_unspendables_unclaimed_rewards += unclaimed_rewards; + + std::pair<uint256, DBVal> value; + value.first = pindex->GetBlockHash(); + value.second.transaction_output_count = m_transaction_output_count; + value.second.bogo_size = m_bogo_size; + value.second.total_amount = m_total_amount; + value.second.total_subsidy = m_total_subsidy; + value.second.block_unspendable_amount = m_block_unspendable_amount; + value.second.block_prevout_spent_amount = m_block_prevout_spent_amount; + value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount; + value.second.block_coinbase_amount = m_block_coinbase_amount; + value.second.unspendables_genesis_block = m_unspendables_genesis_block; + value.second.unspendables_bip30 = m_unspendables_bip30; + value.second.unspendables_scripts = m_unspendables_scripts; + value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards; + + uint256 out; + m_muhash.Finalize(out); + value.second.muhash = out; + + return m_db->Write(DBHeightKey(pindex->nHeight), value) && m_db->Write(DB_MUHASH, m_muhash); +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key{start_height}; + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair<uint256, DBVal> value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); + + // During a reorg, we need to copy all hash digests for blocks that are + // getting disconnected from the height index to the hash index so we can + // still find them when the height index entries are overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + if (!m_db->WriteBatch(batch)) return false; + + { + LOCK(cs_main); + CBlockIndex* iter_tip{g_chainman.m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const auto& consensus_params{Params().GetConsensus()}; + + do { + CBlock block; + + if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + return error("%s: Failed to read block %s from disk", + __func__, iter_tip->GetBlockHash().ToString()); + } + + ReverseBlock(block, iter_tip); + + iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); + } while (new_tip != iter_tip); + } + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookUpOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value + // there matches the block hash. This should be the case if the block is on + // the active chain. + std::pair<uint256, DBVal> read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the + // result will be stored in the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const +{ + DBVal entry; + if (!LookUpOne(*m_db, block_index, entry)) { + return false; + } + + coins_stats.hashSerialized = entry.muhash; + coins_stats.nTransactionOutputs = entry.transaction_output_count; + coins_stats.nBogoSize = entry.bogo_size; + coins_stats.nTotalAmount = entry.total_amount; + coins_stats.total_subsidy = entry.total_subsidy; + coins_stats.block_unspendable_amount = entry.block_unspendable_amount; + coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount; + coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + coins_stats.block_coinbase_amount = entry.block_coinbase_amount; + coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block; + coins_stats.unspendables_bip30 = entry.unspendables_bip30; + coins_stats.unspendables_scripts = entry.unspendables_scripts; + coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + + return true; +} + +bool CoinStatsIndex::Init() +{ + if (!m_db->Read(DB_MUHASH, m_muhash)) { + // Check that the cause of the read failure is that the key does not + // exist. Any other errors indicate database corruption or a disk + // failure, and starting the index would cause further corruption. + if (m_db->Exists(DB_MUHASH)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + } + + if (BaseIndex::Init()) { + const CBlockIndex* pindex{CurrentIndex()}; + + if (pindex) { + DBVal entry; + if (!LookUpOne(*m_db, pindex, entry)) { + return false; + } + + m_transaction_output_count = entry.transaction_output_count; + m_bogo_size = entry.bogo_size; + m_total_amount = entry.total_amount; + m_total_subsidy = entry.total_subsidy; + m_block_unspendable_amount = entry.block_unspendable_amount; + m_block_prevout_spent_amount = entry.block_prevout_spent_amount; + m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + m_block_coinbase_amount = entry.block_coinbase_amount; + m_unspendables_genesis_block = entry.unspendables_genesis_block; + m_unspendables_bip30 = entry.unspendables_bip30; + m_unspendables_scripts = entry.unspendables_scripts; + m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + } + + return true; + } + + return false; +} + +// Reverse a single block as part of a reorg +bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + std::pair<uint256, DBVal> read_out; + + const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + m_total_subsidy -= block_subsidy; + + // Ignore genesis block + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + if (read_out.first != expected_block_hash) { + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + } + } + + // Remove the new UTXOs that were created from the block + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx{block.vtx.at(i)}; + + for (size_t j = 0; j < tx->vout.size(); ++j) { + const CTxOut& out{tx->vout[j]}; + COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)}; + Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + + // Skip unspendable coins + if (coin.out.scriptPubKey.IsUnspendable()) { + m_block_unspendable_amount -= coin.out.nValue; + m_unspendables_scripts -= coin.out.nValue; + continue; + } + + m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); + + if (tx->IsCoinBase()) { + m_block_coinbase_amount -= coin.out.nValue; + } else { + m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue; + } + + --m_transaction_output_count; + m_total_amount -= coin.out.nValue; + m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); + } + + // The coinbase tx has no undo data since no former output is spent + if (!tx->IsCoinBase()) { + const auto& tx_undo{block_undo.vtxundo.at(i - 1)}; + + for (size_t j = 0; j < tx_undo.vprevout.size(); ++j) { + Coin coin{tx_undo.vprevout[j]}; + COutPoint outpoint{tx->vin[j].prevout.hash, tx->vin[j].prevout.n}; + + m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + + m_block_prevout_spent_amount -= coin.out.nValue; + + m_transaction_output_count++; + m_total_amount += coin.out.nValue; + m_bogo_size += GetBogoSize(coin.out.scriptPubKey); + } + } + } + + const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)}; + m_block_unspendable_amount -= unclaimed_rewards; + m_unspendables_unclaimed_rewards -= unclaimed_rewards; + + // Check that the rolled back internal values are consistent with the DB read out + uint256 out; + m_muhash.Finalize(out); + Assert(read_out.second.muhash == out); + + Assert(m_transaction_output_count == read_out.second.transaction_output_count); + Assert(m_total_amount == read_out.second.total_amount); + Assert(m_bogo_size == read_out.second.bogo_size); + Assert(m_total_subsidy == read_out.second.total_subsidy); + Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount); + Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount); + Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount); + Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount); + Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block); + Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30); + Assert(m_unspendables_scripts == read_out.second.unspendables_scripts); + Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards); + + return m_db->Write(DB_MUHASH, m_muhash); +} diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h new file mode 100644 index 0000000000..6149f9b4b3 --- /dev/null +++ b/src/index/coinstatsindex.h @@ -0,0 +1,61 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_COINSTATSINDEX_H +#define BITCOIN_INDEX_COINSTATSINDEX_H + +#include <chain.h> +#include <crypto/muhash.h> +#include <flatfile.h> +#include <index/base.h> +#include <node/coinstats.h> + +/** + * CoinStatsIndex maintains statistics on the UTXO set. + */ +class CoinStatsIndex final : public BaseIndex +{ +private: + std::string m_name; + std::unique_ptr<BaseIndex::DB> m_db; + + MuHash3072 m_muhash; + uint64_t m_transaction_output_count{0}; + uint64_t m_bogo_size{0}; + CAmount m_total_amount{0}; + CAmount m_total_subsidy{0}; + CAmount m_block_unspendable_amount{0}; + CAmount m_block_prevout_spent_amount{0}; + CAmount m_block_new_outputs_ex_coinbase_amount{0}; + CAmount m_block_coinbase_amount{0}; + CAmount m_unspendables_genesis_block{0}; + CAmount m_unspendables_bip30{0}; + CAmount m_unspendables_scripts{0}; + CAmount m_unspendables_unclaimed_rewards{0}; + + bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); + +protected: + bool Init() override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return "coinstatsindex"; } + +public: + // Constructs the index, which becomes available to be queried. + explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + // Look up stats for a specific block using CBlockIndex + bool LookUpStats(const CBlockIndex* block_index, CCoinsStats& coins_stats) const; +}; + +/// The global UTXO set hash object. +extern std::unique_ptr<CoinStatsIndex> g_coin_stats_index; + +#endif // BITCOIN_INDEX_COINSTATSINDEX_H diff --git a/src/init.cpp b/src/init.cpp index 24d67f48dc..10404f9887 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -21,6 +21,7 @@ #include <httprpc.h> #include <httpserver.h> #include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> #include <index/txindex.h> #include <init/common.h> #include <interfaces/chain.h> @@ -165,6 +166,9 @@ void Interrupt(NodeContext& node) g_txindex->Interrupt(); } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); + if (g_coin_stats_index) { + g_coin_stats_index->Interrupt(); + } } void Shutdown(NodeContext& node) @@ -237,6 +241,10 @@ void Shutdown(NodeContext& node) g_txindex->Stop(); g_txindex.reset(); } + if (g_coin_stats_index) { + g_coin_stats_index->Stop(); + g_coin_stats_index.reset(); + } ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); DestroyAllBlockFilterIndexes(); @@ -376,6 +384,7 @@ void SetupServerArgs(NodeContext& node) #endif argsman.AddArg("-blockreconstructionextratxn=<n>", strprintf("Extra transactions to keep in memory for compact block reconstructions (default: %u)", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksonly", strprintf("Whether to reject transactions from network peers. Automatic broadcast and rebroadcast of any transactions from inbound peers is disabled, unless the peer has the 'forcerelay' permission. RPC transactions are not affected. (default: %u)", DEFAULT_BLOCKSONLY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-coinstatsindex", strprintf("Maintain coinstats index used by the gettxoutsetinfo RPC (default: %u)", DEFAULT_COINSTATSINDEX), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); @@ -391,7 +400,7 @@ void SetupServerArgs(NodeContext& node) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " + argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -876,10 +885,12 @@ bool AppInitParameterInteraction(const ArgsManager& args) nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); } - // if using block pruning, then disallow txindex + // if using block pruning, then disallow txindex and coinstatsindex if (args.GetArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) + return InitError(_("Prune mode is incompatible with -coinstatsindex.")); } // -bind and -whitebind can't be set when not listening @@ -1525,11 +1536,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) break; } - // Only verify the DB of the active chainstate. This is fixed in later - // work when we allow VerifyDB to be parameterized by chainstate. - if (&::ChainstateActive() == chainstate && - !CVerifyDB().VerifyDB( - chainparams, *chainstate, &chainstate->CoinsDB(), + if (!CVerifyDB().VerifyDB( + *chainstate, chainparams, chainstate->CoinsDB(), args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); @@ -1590,6 +1598,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) GetBlockFilterIndex(filter_type)->Start(); } + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { + g_coin_stats_index = std::make_unique<CoinStatsIndex>(/* cache size */ 0, false, fReindex); + g_coin_stats_index->Start(); + } + // ********************************************************* Step 9: load wallet for (const auto& client : node.chain_clients) { if (!client->load()) { diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index f8f0fff43f..38c1d29250 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -8,6 +8,7 @@ #include <coins.h> #include <crypto/muhash.h> #include <hash.h> +#include <index/coinstatsindex.h> #include <serialize.h> #include <uint256.h> #include <util/system.h> @@ -16,44 +17,22 @@ #include <map> // Database-independent metric indicating the UTXO set size -static uint64_t GetBogoSize(const CScript& scriptPubKey) +uint64_t GetBogoSize(const CScript& script_pub_key) { return 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + 2 /* scriptPubKey len */ + - scriptPubKey.size() /* scriptPubKey */; + script_pub_key.size() /* scriptPubKey */; } -static void ApplyHash(CCoinsStats& stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) -{ - if (it == outputs.begin()) { - ss << hash; - ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); - } - - ss << VARINT(it->first + 1); - ss << it->second.out.scriptPubKey; - ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - - if (it == std::prev(outputs.end())) { - ss << VARINT(0u); - } -} - -static void ApplyHash(CCoinsStats& stats, std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) {} - -static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs, std::map<uint32_t, Coin>::const_iterator it) -{ - COutPoint outpoint = COutPoint(hash, it->first); - Coin coin = it->second; - +CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin) { CDataStream ss(SER_DISK, PROTOCOL_VERSION); ss << outpoint; ss << static_cast<uint32_t>(coin.nHeight * 2 + coin.fCoinBase); ss << coin.out; - muhash.Insert(MakeUCharSpan(ss)); + return ss; } //! Warning: be very careful when changing this! assumeutxo and UTXO snapshot @@ -68,14 +47,40 @@ static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& has //! It is also possible, though very unlikely, that a change in this //! construction could cause a previously invalid (and potentially malicious) //! UTXO snapshot to be considered valid. -template <typename T> -static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + if (it == outputs.begin()) { + ss << hash; + ss << VARINT(it->second.nHeight * 2 + it->second.fCoinBase ? 1u : 0u); + } + + ss << VARINT(it->first + 1); + ss << it->second.out.scriptPubKey; + ss << VARINT_MODE(it->second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + + if (it == std::prev(outputs.end())) { + ss << VARINT(0u); + } + } +} + +static void ApplyHash(std::nullptr_t, const uint256& hash, const std::map<uint32_t, Coin>& outputs) {} + +static void ApplyHash(MuHash3072& muhash, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + for (auto it = outputs.begin(); it != outputs.end(); ++it) { + COutPoint outpoint = COutPoint(hash, it->first); + Coin coin = it->second; + muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); + } +} + +static void ApplyStats(CCoinsStats& stats, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { assert(!outputs.empty()); stats.nTransactions++; for (auto it = outputs.begin(); it != outputs.end(); ++it) { - ApplyHash(stats, hash_obj, hash, outputs, it); - stats.nTransactionOutputs++; stats.nTotalAmount += it->second.out.nValue; stats.nBogoSize += GetBogoSize(it->second.out.scriptPubKey); @@ -84,18 +89,25 @@ static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, con //! Calculate statistics about the unspent transaction output set template <typename T> -static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point) +static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, T hash_obj, const std::function<void()>& interruption_point, const CBlockIndex* pindex) { - stats = CCoinsStats(); std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); - stats.hashBlock = pcursor->GetBestBlock(); - { - LOCK(cs_main); - assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); - const CBlockIndex* block = blockman.LookupBlockIndex(stats.hashBlock); - stats.nHeight = Assert(block)->nHeight; + if (!pindex) { + { + LOCK(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); + pindex = blockman.LookupBlockIndex(view->GetBestBlock()); + } + } + stats.nHeight = Assert(pindex)->nHeight; + stats.hashBlock = pindex->GetBlockHash(); + + // Use CoinStatsIndex if it is requested and available and a hash_type of Muhash or None was requested + if ((stats.m_hash_type == CoinStatsHashType::MUHASH || stats.m_hash_type == CoinStatsHashType::NONE) && g_coin_stats_index && stats.index_requested) { + stats.index_used = true; + return g_coin_stats_index->LookUpStats(pindex, stats); } PrepareHash(hash_obj, stats); @@ -108,7 +120,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, hash_obj, prevkey, outputs); + ApplyStats(stats, prevkey, outputs); + ApplyHash(hash_obj, prevkey, outputs); outputs.clear(); } prevkey = key.hash; @@ -120,7 +133,8 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& pcursor->Next(); } if (!outputs.empty()) { - ApplyStats(stats, hash_obj, prevkey, outputs); + ApplyStats(stats, prevkey, outputs); + ApplyHash(hash_obj, prevkey, outputs); } FinalizeHash(hash_obj, stats); @@ -129,19 +143,19 @@ static bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& return true; } -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, CoinStatsHashType hash_type, const std::function<void()>& interruption_point) +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point, const CBlockIndex* pindex) { - switch (hash_type) { + switch (stats.m_hash_type) { case(CoinStatsHashType::HASH_SERIALIZED): { CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - return GetUTXOStats(view, blockman, stats, ss, interruption_point); + return GetUTXOStats(view, blockman, stats, ss, interruption_point, pindex); } case(CoinStatsHashType::MUHASH): { MuHash3072 muhash; - return GetUTXOStats(view, blockman, stats, muhash, interruption_point); + return GetUTXOStats(view, blockman, stats, muhash, interruption_point, pindex); } case(CoinStatsHashType::NONE): { - return GetUTXOStats(view, blockman, stats, nullptr, interruption_point); + return GetUTXOStats(view, blockman, stats, nullptr, interruption_point, pindex); } } // no default case, so the compiler can warn about missing cases assert(false); diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 975651dcc4..8be256edc9 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -7,6 +7,9 @@ #define BITCOIN_NODE_COINSTATS_H #include <amount.h> +#include <chain.h> +#include <coins.h> +#include <streams.h> #include <uint256.h> #include <cstdint> @@ -23,6 +26,7 @@ enum class CoinStatsHashType { struct CCoinsStats { + CoinStatsHashType m_hash_type; int nHeight{0}; uint256 hashBlock{}; uint64_t nTransactions{0}; @@ -34,9 +38,31 @@ struct CCoinsStats //! The number of coins contained. uint64_t coins_count{0}; + + //! Signals if the coinstatsindex should be used (when available). + bool index_requested{true}; + //! Signals if the coinstatsindex was used to retrieve the statistics. + bool index_used{false}; + + // Following values are only available from coinstats index + CAmount total_subsidy{0}; + CAmount block_unspendable_amount{0}; + CAmount block_prevout_spent_amount{0}; + CAmount block_new_outputs_ex_coinbase_amount{0}; + CAmount block_coinbase_amount{0}; + CAmount unspendables_genesis_block{0}; + CAmount unspendables_bip30{0}; + CAmount unspendables_scripts{0}; + CAmount unspendables_unclaimed_rewards{0}; + + CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; //! Calculate statistics about the unspent transaction output set -bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const CoinStatsHashType hash_type, const std::function<void()>& interruption_point = {}); +bool GetUTXOStats(CCoinsView* view, BlockManager& blockman, CCoinsStats& stats, const std::function<void()>& interruption_point = {}, const CBlockIndex* pindex = nullptr); + +uint64_t GetBogoSize(const CScript& script_pub_key); + +CDataStream TxOutSer(const COutPoint& outpoint, const Coin& coin); #endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index de71b7dea7..a30cac3504 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -319,11 +319,9 @@ void BitcoinApplication::parameterSetup() InitParameterInteraction(gArgs); } -void BitcoinApplication::InitializePruneSetting(bool prune) +void BitcoinApplication::InitPruneSetting(int64_t prune_MiB) { - // If prune is set, intentionally override existing prune size with - // the default size since this is called when choosing a new datadir. - optionsModel->SetPruneTargetGB(prune ? DEFAULT_PRUNE_TARGET_GB : 0, true); + optionsModel->SetPruneTargetGB(PruneMiBtoGB(prune_MiB), true); } void BitcoinApplication::requestInitialize() @@ -533,9 +531,9 @@ int GuiMain(int argc, char* argv[]) /// 5. Now that settings and translations are available, ask user for data directory // User language is set up: pick a data directory bool did_show_intro = false; - bool prune = false; // Intro dialog prune check box + int64_t prune_MiB = 0; // Intro dialog prune configuration // Gracefully exit if the user cancels - if (!Intro::showIfNeeded(did_show_intro, prune)) return EXIT_SUCCESS; + if (!Intro::showIfNeeded(did_show_intro, prune_MiB)) return EXIT_SUCCESS; /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes @@ -617,7 +615,7 @@ int GuiMain(int argc, char* argv[]) if (did_show_intro) { // Store intro dialog settings other than datadir (network specific) - app.InitializePruneSetting(prune); + app.InitPruneSetting(prune_MiB); } if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 5fd6bd607f..f9fab0534b 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -68,7 +68,7 @@ public: /// Create options model void createOptionsModel(bool resetSettings); /// Initialize prune setting - void InitializePruneSetting(bool prune); + void InitPruneSetting(int64_t prune_MiB); /// Create main window void createWindow(const NetworkStyle *networkStyle); /// Create splash screen diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index f2c555de52..04161020b2 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -8,6 +8,7 @@ #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/peertablemodel.h> +#include <qt/peertablesortproxy.h> #include <clientversion.h> #include <interfaces/handler.h> @@ -38,7 +39,11 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO { cachedBestHeaderHeight = -1; cachedBestHeaderTime = -1; + peerTableModel = new PeerTableModel(m_node, this); + m_peer_table_sort_proxy = new PeerTableSortProxy(this); + m_peer_table_sort_proxy->setSourceModel(peerTableModel); + banTableModel = new BanTableModel(m_node, this); QTimer* timer = new QTimer; @@ -184,6 +189,11 @@ PeerTableModel *ClientModel::getPeerTableModel() return peerTableModel; } +PeerTableSortProxy* ClientModel::peerTableSortProxy() +{ + return m_peer_table_sort_proxy; +} + BanTableModel *ClientModel::getBanTableModel() { return banTableModel; diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 7ac4cc040b..7a199ef19c 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -17,6 +17,7 @@ class BanTableModel; class CBlockIndex; class OptionsModel; class PeerTableModel; +class PeerTableSortProxy; enum class SynchronizationState; namespace interfaces { @@ -54,6 +55,7 @@ public: interfaces::Node& node() const { return m_node; } OptionsModel *getOptionsModel(); PeerTableModel *getPeerTableModel(); + PeerTableSortProxy* peerTableSortProxy(); BanTableModel *getBanTableModel(); //! Return number of connections, default is in- and outbound (total) @@ -96,6 +98,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_notify_header_tip; OptionsModel *optionsModel; PeerTableModel *peerTableModel; + PeerTableSortProxy* m_peer_table_sort_proxy{nullptr}; BanTableModel *banTableModel; //! A thread to interact with m_node asynchronously diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 13687a6510..61ecea1cac 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -923,6 +923,9 @@ <property name="tabKeyNavigation"> <bool>false</bool> </property> + <property name="alternatingRowColors"> + <bool>true</bool> + </property> <property name="textElideMode"> <enum>Qt::ElideMiddle</enum> </property> @@ -984,6 +987,9 @@ <property name="tabKeyNavigation"> <bool>false</bool> </property> + <property name="alternatingRowColors"> + <bool>true</bool> + </property> <property name="sortingEnabled"> <bool>true</bool> </property> diff --git a/src/qt/forms/intro.ui b/src/qt/forms/intro.ui index f27a4ebe44..a1e94f99e6 100644 --- a/src/qt/forms/intro.ui +++ b/src/qt/forms/intro.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>674</width> - <height>415</height> + <height>447</height> </rect> </property> <property name="windowTitle"> @@ -211,16 +211,6 @@ </widget> </item> <item> - <widget class="QCheckBox" name="prune"> - <property name="toolTip"> - <string>Reverting this setting requires re-downloading the entire blockchain. It is faster to download the full chain first and prune it later. Disables some advanced features.</string> - </property> - <property name="text"> - <string></string> - </property> - </widget> - </item> - <item> <widget class="QLabel" name="lblExplanation2"> <property name="text"> <string>This initial synchronisation is very demanding, and may expose hardware problems with your computer that had previously gone unnoticed. Each time you run %1, it will continue downloading where it left off.</string> @@ -241,6 +231,47 @@ </widget> </item> <item> + <layout class="QHBoxLayout" name="pruneOptLayout"> + <item> + <widget class="QCheckBox" name="prune"> + <property name="text"> + <string>Limit block chain storage to</string> + </property> + <property name="toolTip"> + <string>Reverting this setting requires re-downloading the entire blockchain. It is faster to download the full chain first and prune it later. Disables some advanced features.</string> + </property> + </widget> + </item> + <item> + <widget class="QSpinBox" name="pruneGB"> + <property name="suffix"> + <string> GB</string> + </property> + </widget> + </item> + <item> + <widget class="QLabel" name="lblPruneSuffix"> + <property name="buddy"> + <cstring>pruneGB</cstring> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer_2"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> + <item> <spacer name="verticalSpacer"> <property name="orientation"> <enum>Qt::Vertical</enum> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index d2e7ca8f06..83b6a59aa0 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -271,16 +271,6 @@ QLabel { color: rgb(40,40,40); }</string> </property> </widget> </item> - <item> - <widget class="QProgressBar" name="progressBar"> - <property name="value"> - <number>24</number> - </property> - <property name="format"> - <string/> - </property> - </widget> - </item> </layout> </item> <item row="4" column="0"> diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index aa6b2665fa..ed39307fd7 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -17,6 +17,7 @@ #include <interfaces/node.h> #include <util/system.h> +#include <validation.h> #include <QFileDialog> #include <QSettings> @@ -139,17 +140,26 @@ Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_si ); ui->lblExplanation2->setText(ui->lblExplanation2->text().arg(PACKAGE_NAME)); + const int min_prune_target_GB = std::ceil(MIN_DISK_SPACE_FOR_BLOCK_FILES / 1e9); + ui->pruneGB->setRange(min_prune_target_GB, std::numeric_limits<int>::max()); if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } - ui->prune->setText(tr("Discard blocks after verification, except most recent %1 GB (prune)").arg(m_prune_target_gb)); + ui->pruneGB->setValue(m_prune_target_gb); + ui->pruneGB->setToolTip(ui->prune->toolTip()); + ui->lblPruneSuffix->setToolTip(ui->prune->toolTip()); UpdatePruneLabels(ui->prune->isChecked()); connect(ui->prune, &QCheckBox::toggled, [this](bool prune_checked) { UpdatePruneLabels(prune_checked); UpdateFreeSpaceLabel(); }); + connect(ui->pruneGB, QOverload<int>::of(&QSpinBox::valueChanged), [this](int prune_GB) { + m_prune_target_gb = prune_GB; + UpdatePruneLabels(ui->prune->isChecked()); + UpdateFreeSpaceLabel(); + }); startThread(); } @@ -182,7 +192,17 @@ void Intro::setDataDirectory(const QString &dataDir) } } -bool Intro::showIfNeeded(bool& did_show_intro, bool& prune) +int64_t Intro::getPruneMiB() const +{ + switch (ui->prune->checkState()) { + case Qt::Checked: + return PruneGBtoMiB(m_prune_target_gb); + case Qt::Unchecked: default: + return 0; + } +} + +bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) { did_show_intro = false; @@ -233,7 +253,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, bool& prune) } // Additional preferences: - prune = intro.ui->prune->isChecked(); + prune_MiB = intro.getPruneMiB(); settings.setValue("strDataDir", dataDir); settings.setValue("fReset", false); @@ -361,6 +381,11 @@ void Intro::UpdatePruneLabels(bool prune_checked) storageRequiresMsg = tr("Approximately %1 GB of data will be stored in this directory."); } ui->lblExplanation3->setVisible(prune_checked); + ui->pruneGB->setEnabled(prune_checked); + static constexpr uint64_t nPowTargetSpacing = 10 * 60; // from chainparams, which we don't have at this stage + static constexpr uint32_t expected_block_data_size = 2250000; // includes undo data + const uint64_t expected_backup_days = m_prune_target_gb * 1e9 / (uint64_t(expected_block_data_size) * 86400 / nPowTargetSpacing); + ui->lblPruneSuffix->setText(tr("(sufficient to restore backups %n day(s) old)", "block chain pruning", expected_backup_days)); ui->sizeWarningLabel->setText( tr("%1 will download and store a copy of the Bitcoin block chain.").arg(PACKAGE_NAME) + " " + storageRequiresMsg.arg(m_required_space_gb) + " " + diff --git a/src/qt/intro.h b/src/qt/intro.h index 51f42de7ac..88fe2b722d 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -36,6 +36,7 @@ public: QString getDataDirectory(); void setDataDirectory(const QString &dataDir); + int64_t getPruneMiB() const; /** * Determine data directory. Let the user choose if the current one doesn't exist. @@ -47,7 +48,7 @@ public: * @note do NOT call global GetDataDir() before calling this function, this * will cause the wrong path to be cached. */ - static bool showIfNeeded(bool& did_show_intro, bool& prune); + static bool showIfNeeded(bool& did_show_intro, int64_t& prune_MiB); Q_SIGNALS: void requestCheck(); @@ -72,7 +73,7 @@ private: //! Total required space (in GB) depending on user choice (prune or not prune). int64_t m_required_space_gb{0}; uint64_t m_bytes_available{0}; - const int64_t m_prune_target_gb; + int64_t m_prune_target_gb; void startThread(); void checkPath(const QString &dataDir); diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 8070aa627c..6363fe2edf 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -134,7 +134,6 @@ void ModalOverlay::tipUpdate(int count, const QDateTime& blockDate, double nVeri // show the percentage done according to nVerificationProgress ui->percentageProgress->setText(QString::number(nVerificationProgress*100, 'f', 2)+"%"); - ui->progressBar->setValue(nVerificationProgress*100); if (!bestHeaderDate.isValid()) // not syncing diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 3459bf4cf8..6c4e326011 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -11,56 +11,19 @@ #include <utility> -#include <QDebug> #include <QList> #include <QTimer> -bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombinedStats &right) const -{ - const CNodeStats *pLeft = &(left.nodeStats); - const CNodeStats *pRight = &(right.nodeStats); - - if (order == Qt::DescendingOrder) - std::swap(pLeft, pRight); - - switch (static_cast<PeerTableModel::ColumnIndex>(column)) { - case PeerTableModel::NetNodeId: - return pLeft->nodeid < pRight->nodeid; - case PeerTableModel::Address: - return pLeft->addrName.compare(pRight->addrName) < 0; - case PeerTableModel::ConnectionType: - return pLeft->m_conn_type < pRight->m_conn_type; - case PeerTableModel::Network: - return pLeft->m_network < pRight->m_network; - case PeerTableModel::Ping: - return pLeft->m_min_ping_time < pRight->m_min_ping_time; - case PeerTableModel::Sent: - return pLeft->nSendBytes < pRight->nSendBytes; - case PeerTableModel::Received: - return pLeft->nRecvBytes < pRight->nRecvBytes; - case PeerTableModel::Subversion: - return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; - } // no default case, so the compiler can warn about missing cases - assert(false); -} - // private implementation class PeerTablePriv { public: /** Local cache of peer information */ QList<CNodeCombinedStats> cachedNodeStats; - /** Column to sort nodes by (default to unsorted) */ - int sortColumn{-1}; - /** Order (ascending or descending) to sort nodes by */ - Qt::SortOrder sortOrder; - /** Index of rows by node ID */ - std::map<NodeId, int> mapNodeRows; /** Pull a full list of peers from vNodes into our cache */ void refreshPeers(interfaces::Node& node) { - { cachedNodeStats.clear(); interfaces::Node::NodesStats nodes_stats; @@ -74,17 +37,6 @@ public: stats.nodeStateStats = std::get<2>(node_stats); cachedNodeStats.append(stats); } - } - - if (sortColumn >= 0) - // sort cacheNodeStats (use stable sort to prevent rows jumping around unnecessarily) - std::stable_sort(cachedNodeStats.begin(), cachedNodeStats.end(), NodeLessThan(sortColumn, sortOrder)); - - // build index map - mapNodeRows.clear(); - int row = 0; - for (const CNodeCombinedStats& stats : cachedNodeStats) - mapNodeRows.insert(std::pair<NodeId, int>(stats.nodeStats.nodeid, row++)); } int size() const @@ -194,10 +146,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const } // no default case, so the compiler can warn about missing cases assert(false); } else if (role == StatsRole) { - switch (index.column()) { - case NetNodeId: return QVariant::fromValue(rec); - default: return QVariant(); - } + return QVariant::fromValue(rec); } return QVariant(); @@ -239,19 +188,3 @@ void PeerTableModel::refresh() priv->refreshPeers(m_node); Q_EMIT layoutChanged(); } - -int PeerTableModel::getRowByNodeId(NodeId nodeid) -{ - std::map<NodeId, int>::iterator it = priv->mapNodeRows.find(nodeid); - if (it == priv->mapNodeRows.end()) - return -1; - - return it->second; -} - -void PeerTableModel::sort(int column, Qt::SortOrder order) -{ - priv->sortColumn = column; - priv->sortOrder = order; - refresh(); -} diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0823235ec0..9c7bc25da2 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -30,18 +30,6 @@ struct CNodeCombinedStats { }; Q_DECLARE_METATYPE(CNodeCombinedStats*) -class NodeLessThan -{ -public: - NodeLessThan(int nColumn, Qt::SortOrder fOrder) : - column(nColumn), order(fOrder) {} - bool operator()(const CNodeCombinedStats &left, const CNodeCombinedStats &right) const; - -private: - int column; - Qt::SortOrder order; -}; - /** Qt model providing information about connected peers, similar to the "getpeerinfo" RPC call. Used by the rpc console UI. @@ -53,7 +41,6 @@ class PeerTableModel : public QAbstractTableModel public: explicit PeerTableModel(interfaces::Node& node, QObject* parent); ~PeerTableModel(); - int getRowByNodeId(NodeId nodeid); void startAutoRefresh(); void stopAutoRefresh(); @@ -80,7 +67,6 @@ public: QVariant headerData(int section, Qt::Orientation orientation, int role) const override; QModelIndex index(int row, int column, const QModelIndex &parent) const override; Qt::ItemFlags flags(const QModelIndex &index) const override; - void sort(int column, Qt::SortOrder order) override; /*@}*/ public Q_SLOTS: diff --git a/src/qt/peertablesortproxy.cpp b/src/qt/peertablesortproxy.cpp new file mode 100644 index 0000000000..78932da8d4 --- /dev/null +++ b/src/qt/peertablesortproxy.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <qt/peertablesortproxy.h> + +#include <qt/peertablemodel.h> +#include <util/check.h> + +#include <QModelIndex> +#include <QString> +#include <QVariant> + +PeerTableSortProxy::PeerTableSortProxy(QObject* parent) + : QSortFilterProxyModel(parent) +{ +} + +bool PeerTableSortProxy::lessThan(const QModelIndex& left_index, const QModelIndex& right_index) const +{ + const CNodeStats left_stats = Assert(sourceModel()->data(left_index, PeerTableModel::StatsRole).value<CNodeCombinedStats*>())->nodeStats; + const CNodeStats right_stats = Assert(sourceModel()->data(right_index, PeerTableModel::StatsRole).value<CNodeCombinedStats*>())->nodeStats; + + switch (static_cast<PeerTableModel::ColumnIndex>(left_index.column())) { + case PeerTableModel::NetNodeId: + return left_stats.nodeid < right_stats.nodeid; + case PeerTableModel::Address: + return left_stats.addrName.compare(right_stats.addrName) < 0; + case PeerTableModel::ConnectionType: + return left_stats.m_conn_type < right_stats.m_conn_type; + case PeerTableModel::Network: + return left_stats.m_network < right_stats.m_network; + case PeerTableModel::Ping: + return left_stats.m_min_ping_time < right_stats.m_min_ping_time; + case PeerTableModel::Sent: + return left_stats.nSendBytes < right_stats.nSendBytes; + case PeerTableModel::Received: + return left_stats.nRecvBytes < right_stats.nRecvBytes; + case PeerTableModel::Subversion: + return left_stats.cleanSubVer.compare(right_stats.cleanSubVer) < 0; + } // no default case, so the compiler can warn about missing cases + assert(false); +} diff --git a/src/qt/peertablesortproxy.h b/src/qt/peertablesortproxy.h new file mode 100644 index 0000000000..1879f6b400 --- /dev/null +++ b/src/qt/peertablesortproxy.h @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_PEERTABLESORTPROXY_H +#define BITCOIN_QT_PEERTABLESORTPROXY_H + +#include <QSortFilterProxyModel> + +QT_BEGIN_NAMESPACE +class QModelIndex; +QT_END_NAMESPACE + +class PeerTableSortProxy : public QSortFilterProxyModel +{ + Q_OBJECT + +public: + explicit PeerTableSortProxy(QObject* parent = nullptr); + +protected: + bool lessThan(const QModelIndex& left_index, const QModelIndex& right_index) const override; +}; + +#endif // BITCOIN_QT_PEERTABLESORTPROXY_H diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 85412ca75b..006f60e7a1 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -9,13 +9,14 @@ #include <qt/rpcconsole.h> #include <qt/forms/ui_debugwindow.h> +#include <chainparams.h> +#include <interfaces/node.h> +#include <netbase.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> +#include <qt/peertablesortproxy.h> #include <qt/platformstyle.h> #include <qt/walletmodel.h> -#include <chainparams.h> -#include <interfaces/node.h> -#include <netbase.h> #include <rpc/client.h> #include <rpc/server.h> #include <util/strencodings.h> @@ -606,7 +607,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ connect(model, &ClientModel::mempoolSizeChanged, this, &RPCConsole::setMempoolSize); // set up peer table - ui->peerWidget->setModel(model->getPeerTableModel()); + ui->peerWidget->setModel(model->peerTableSortProxy()); ui->peerWidget->verticalHeader()->hide(); ui->peerWidget->setSelectionBehavior(QAbstractItemView::SelectRows); ui->peerWidget->setSelectionMode(QAbstractItemView::ExtendedSelection); @@ -627,10 +628,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // peer table signal handling - update peer details when selecting new node connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::updateDetailWidget); - // peer table signal handling - update peer details when new nodes are added to the model - connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::peerLayoutChanged); - // peer table signal handling - cache selected node ids - connect(model->getPeerTableModel(), &PeerTableModel::layoutAboutToBeChanged, this, &RPCConsole::peerLayoutAboutToChange); + connect(model->getPeerTableModel(), &PeerTableModel::layoutChanged, this, &RPCConsole::updateDetailWidget); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); @@ -1014,67 +1012,6 @@ void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } -void RPCConsole::peerLayoutAboutToChange() -{ - cachedNodeids.clear(); - for (const QModelIndex& peer : GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId)) { - const auto stats = peer.data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); - cachedNodeids.append(stats->nodeStats.nodeid); - } -} - -void RPCConsole::peerLayoutChanged() -{ - if (!clientModel || !clientModel->getPeerTableModel()) - return; - - bool fUnselect = false; - bool fReselect = false; - - if (cachedNodeids.empty()) // no node selected yet - return; - - // find the currently selected row - int selectedRow = -1; - QModelIndexList selectedModelIndex = ui->peerWidget->selectionModel()->selectedIndexes(); - if (!selectedModelIndex.isEmpty()) { - selectedRow = selectedModelIndex.first().row(); - } - - // check if our detail node has a row in the table (it may not necessarily - // be at selectedRow since its position can change after a layout change) - int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.first()); - - if (detailNodeRow < 0) - { - // detail node disappeared from table (node disconnected) - fUnselect = true; - } - else - { - if (detailNodeRow != selectedRow) - { - // detail node moved position - fUnselect = true; - fReselect = true; - } - } - - if (fUnselect && selectedRow >= 0) { - clearSelectedNode(); - } - - if (fReselect) - { - for(int i = 0; i < cachedNodeids.size(); i++) - { - ui->peerWidget->selectRow(clientModel->getPeerTableModel()->getRowByNodeId(cachedNodeids.at(i))); - } - } - - updateDetailWidget(); -} - void RPCConsole::updateDetailWidget() { const QList<QModelIndex> selected_peers = GUIUtil::getEntryData(ui->peerWidget, PeerTableModel::NetNodeId); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index b9806e40c9..5182d60a0d 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -118,10 +118,6 @@ public Q_SLOTS: void browseHistory(int offset); /** Scroll console view to end */ void scrollToEnd(); - /** Handle selection caching before update */ - void peerLayoutAboutToChange(); - /** Handle updated peer information */ - void peerLayoutChanged(); /** Disconnect a selected node on the Peers tab */ void disconnectSelectedNode(); /** Ban a selected node on the Peers tab */ diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e7fd97ee1f..81cb802357 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,6 +14,7 @@ #include <core_io.h> #include <hash.h> #include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> #include <node/blockstorage.h> #include <node/coinstats.h> #include <node/context.h> @@ -139,6 +140,33 @@ static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* b return blockindex == tip ? 1 : -1; } +CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) { + LOCK(::cs_main); + CChain& active_chain = chainman.ActiveChain(); + + if (param.isNum()) { + const int height{param.get_int()}; + if (height < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); + } + const int current_tip{active_chain.Height()}; + if (height > current_tip) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); + } + + return active_chain[height]; + } else { + const uint256 hash{ParseHashV(param, "hash_or_height")}; + CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); + + if (!pindex) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + + return pindex; + } +} + UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) { // Serialize passed information without accessing chain state of the active chain! @@ -1068,50 +1096,88 @@ static RPCHelpMan gettxoutsetinfo() { return RPCHelpMan{"gettxoutsetinfo", "\nReturns statistics about the unspent transaction output set.\n" - "Note this call may take some time.\n", + "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, + {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "height", "The block height (index) of the returned statistics"}, {RPCResult::Type::STR_HEX, "bestblock", "The hash of the block at which these statistics are calculated"}, - {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs"}, {RPCResult::Type::NUM, "txouts", "The number of unspent transaction outputs"}, - {RPCResult::Type::NUM, "bogosize", "A meaningless metric for UTXO set size"}, + {RPCResult::Type::NUM, "bogosize", "Database-independent, meaningless metric indicating the UTXO set size"}, {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, - {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk"}, + {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, + {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"}, + {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, + {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)", + { + {RPCResult::Type::STR_AMOUNT, "prevout_spent", ""}, + {RPCResult::Type::STR_AMOUNT, "coinbase", ""}, + {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""}, + {RPCResult::Type::STR_AMOUNT, "unspendable", ""}, + {RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories", + { + {RPCResult::Type::STR_AMOUNT, "genesis_block", ""}, + {RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"}, + {RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"}, + {RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"}, + }} + }}, }}, RPCExamples{ - HelpExampleCli("gettxoutsetinfo", "") - + HelpExampleRpc("gettxoutsetinfo", "") + HelpExampleCli("gettxoutsetinfo", "") + + HelpExampleCli("gettxoutsetinfo", R"("none")") + + HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") + + HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") + + HelpExampleRpc("gettxoutsetinfo", "") + + HelpExampleRpc("gettxoutsetinfo", R"("none")") + + HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") + + HelpExampleRpc("gettxoutsetinfo", R"("none", "00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09")") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { UniValue ret(UniValue::VOBJ); - CCoinsStats stats; + CBlockIndex* pindex{nullptr}; + const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; + CCoinsStats stats{hash_type}; + stats.index_requested = request.params[2].isNull() || request.params[2].get_bool(); + NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); CChainState& active_chainstate = chainman.ActiveChainstate(); active_chainstate.ForceFlushStateToDisk(); - const CoinStatsHashType hash_type{request.params[0].isNull() ? CoinStatsHashType::HASH_SERIALIZED : ParseHashType(request.params[0].get_str())}; - CCoinsView* coins_view; BlockManager* blockman; { LOCK(::cs_main); coins_view = &active_chainstate.CoinsDB(); blockman = &active_chainstate.m_blockman; + pindex = blockman->LookupBlockIndex(coins_view->GetBestBlock()); } - if (GetUTXOStats(coins_view, *blockman, stats, hash_type, node.rpc_interruption_point)) { + + if (!request.params[1].isNull()) { + if (!g_coin_stats_index) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Querying specific block heights requires coinstatsindex"); + } + + if (stats.m_hash_type == CoinStatsHashType::HASH_SERIALIZED) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block"); + } + + pindex = ParseHashOrHeight(request.params[1], chainman); + } + + if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point, pindex)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); - ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("bogosize", (int64_t)stats.nBogoSize); if (hash_type == CoinStatsHashType::HASH_SERIALIZED) { @@ -1120,9 +1186,42 @@ static RPCHelpMan gettxoutsetinfo() if (hash_type == CoinStatsHashType::MUHASH) { ret.pushKV("muhash", stats.hashSerialized.GetHex()); } - ret.pushKV("disk_size", stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); + if (!stats.index_used) { + ret.pushKV("transactions", static_cast<int64_t>(stats.nTransactions)); + ret.pushKV("disk_size", stats.nDiskSize); + } else { + ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount)); + + CCoinsStats prev_stats{hash_type}; + + if (pindex->nHeight > 0) { + GetUTXOStats(coins_view, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), prev_stats, node.rpc_interruption_point, pindex->pprev); + } + + UniValue block_info(UniValue::VOBJ); + block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount)); + block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount)); + block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount)); + block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount)); + + UniValue unspendables(UniValue::VOBJ); + unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block)); + unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30)); + unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts)); + unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards)); + block_info.pushKV("unspendables", unspendables); + + ret.pushKV("block_info", block_info); + } } else { + if (g_coin_stats_index) { + const IndexSummary summary{g_coin_stats_index->GetSummary()}; + + if (!summary.synced) { + throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to read UTXO set because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); + } + } throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } return ret; @@ -1239,7 +1338,8 @@ static RPCHelpMan verifychain() LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); - return CVerifyDB().VerifyDB(Params(), active_chainstate, &active_chainstate.CoinsTip(), check_level, check_depth); + return CVerifyDB().VerifyDB( + active_chainstate, Params(), active_chainstate.CoinsTip(), check_level, check_depth); }, }; } @@ -1909,31 +2009,7 @@ static RPCHelpMan getblockstats() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - CChain& active_chain = chainman.ActiveChain(); - - CBlockIndex* pindex; - if (request.params[0].isNum()) { - const int height = request.params[0].get_int(); - const int current_tip = active_chain.Height(); - if (height < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d is negative", height)); - } - if (height > current_tip) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Target block height %d after current tip %d", height, current_tip)); - } - - pindex = active_chain[height]; - } else { - const uint256 hash(ParseHashV(request.params[0], "hash_or_height")); - pindex = chainman.m_blockman.LookupBlockIndex(hash); - if (!pindex) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - } - if (!active_chain.Contains(pindex)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); - } - } - + CBlockIndex* pindex{ParseHashOrHeight(request.params[0], chainman)}; CHECK_NONFATAL(pindex != nullptr); std::set<std::string> stats; @@ -2491,7 +2567,7 @@ static RPCHelpMan dumptxoutset() UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile) { std::unique_ptr<CCoinsViewCursor> pcursor; - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::NONE}; CBlockIndex* tip; { @@ -2511,7 +2587,7 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil chainstate.ForceFlushStateToDisk(); - if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { + if (!GetUTXOStats(&chainstate.CoinsDB(), chainstate.m_blockman, stats, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 2b593cd10b..9c8582c7a3 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -127,6 +127,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, + { "gettxoutsetinfo", 1, "hash_or_height" }, + { "gettxoutsetinfo", 2, "use_index"}, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, { "send", 0, "outputs" }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 09b32345a2..0e0da8f22b 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -5,6 +5,7 @@ #include <httpserver.h> #include <index/blockfilterindex.h> +#include <index/coinstatsindex.h> #include <index/txindex.h> #include <interfaces/chain.h> #include <interfaces/echo.h> @@ -729,6 +730,10 @@ static RPCHelpMan getindexinfo() result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); } + if (g_coin_stats_index) { + result.pushKVs(SummaryToJSON(g_coin_stats_index->GetSummary(), index_name)); + } + ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); }); diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp new file mode 100644 index 0000000000..3fc7b72077 --- /dev/null +++ b/src/test/coinstatsindex_tests.cpp @@ -0,0 +1,79 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <index/coinstatsindex.h> +#include <test/util/setup_common.h> +#include <util/time.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +#include <chrono> + + +BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) + +BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) +{ + CoinStatsIndex coin_stats_index{1 << 20, true}; + + CCoinsStats coin_stats{CoinStatsHashType::MUHASH}; + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = ChainActive().Tip(); + } + + // CoinStatsIndex should not be found before it is started. + BOOST_CHECK(!coin_stats_index.LookUpStats(block_index, coin_stats)); + + // BlockUntilSyncedToCurrentChain should return false before CoinStatsIndex + // is started. + BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain()); + + coin_stats_index.Start(); + + // Allow the CoinStatsIndex to catch up with the block index that is syncing + // in a background thread. + const auto timeout = GetTime<std::chrono::seconds>() + 120s; + while (!coin_stats_index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(timeout > GetTime<std::chrono::milliseconds>()); + UninterruptibleSleep(100ms); + } + + // Check that CoinStatsIndex works for genesis block. + const CBlockIndex* genesis_block_index; + { + LOCK(cs_main); + genesis_block_index = ChainActive().Genesis(); + } + BOOST_CHECK(coin_stats_index.LookUpStats(genesis_block_index, coin_stats)); + + // Check that CoinStatsIndex updates with new blocks. + coin_stats_index.LookUpStats(block_index, coin_stats); + + const CScript script_pub_key{CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG}; + std::vector<CMutableTransaction> noTxns; + CreateAndProcessBlock(noTxns, script_pub_key); + + // Let the CoinStatsIndex to catch up again. + BOOST_CHECK(coin_stats_index.BlockUntilSyncedToCurrentChain()); + + CCoinsStats new_coin_stats{CoinStatsHashType::MUHASH}; + const CBlockIndex* new_block_index; + { + LOCK(cs_main); + new_block_index = ChainActive().Tip(); + } + coin_stats_index.LookUpStats(new_block_index, new_coin_stats); + + BOOST_CHECK(block_index != new_block_index); + + // Shutdown sequence (c.f. Shutdown() in init.cpp) + coin_stats_index.Stop(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 4ddbc8338e..7b661a0d1d 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -72,7 +72,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; BOOST_CHECK_EQUAL(script.size(), 25U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); @@ -89,7 +89,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) script << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; BOOST_CHECK_EQUAL(script.size(), 23U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); @@ -107,7 +107,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) BOOST_CHECK_EQUAL(script.size(), 35U); - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); @@ -124,7 +124,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) BOOST_CHECK_EQUAL(script.size(), 67U); // 1 char code + 65 char pubkey + OP_CHECKSIG - std::vector<unsigned char> out; + CompressedScript out; bool done = CompressScript(script, out); BOOST_CHECK_EQUAL(done, true); diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b21d2eae79..21dc80cc8d 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -258,10 +258,10 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) (void)GetTransactionSigOpCost(transaction, coins_view_cache, flags); }, [&] { - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; bool expected_code_path = false; try { - (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), stats, CoinStatsHashType::HASH_SERIALIZED); + (void)GetUTXOStats(&coins_view_cache, WITH_LOCK(::cs_main, return std::ref(g_chainman.m_blockman)), stats); } catch (const std::logic_error&) { expected_code_path = true; } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp new file mode 100644 index 0000000000..dae6f6b6a7 --- /dev/null +++ b/src/test/fuzz/rpc.cpp @@ -0,0 +1,378 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <base58.h> +#include <chainparamsbase.h> +#include <core_io.h> +#include <interfaces/chain.h> +#include <key.h> +#include <key_io.h> +#include <node/context.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <psbt.h> +#include <rpc/blockchain.h> +#include <rpc/client.h> +#include <rpc/request.h> +#include <rpc/server.h> +#include <rpc/util.h> +#include <span.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <univalue.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/time.h> + +#include <cstdint> +#include <iostream> +#include <memory> +#include <optional> +#include <stdexcept> +#include <string> +#include <vector> + +namespace { +struct RPCFuzzTestingSetup : public TestingSetup { + RPCFuzzTestingSetup(const std::string& chain_name, const std::vector<const char*>& extra_args) : TestingSetup{chain_name, extra_args} + { + } + + UniValue CallRPC(const std::string& rpc_method, const std::vector<std::string>& arguments) + { + JSONRPCRequest request; + request.context = &m_node; + request.strMethod = rpc_method; + request.params = RPCConvertValues(rpc_method, arguments); + return tableRPC.execute(request); + } + + std::vector<std::string> GetRPCCommands() const + { + return tableRPC.listCommands(); + } +}; + +RPCFuzzTestingSetup* rpc_testing_setup = nullptr; +std::string g_limit_to_rpc_command; + +// RPC commands which are not appropriate for fuzzing: such as RPC commands +// reading or writing to a filename passed as an RPC parameter, RPC commands +// resulting in network activity, etc. +const std::vector<std::string> RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ + "addconnection", // avoid DNS lookups + "addnode", // avoid DNS lookups + "addpeeraddress", // avoid DNS lookups + "analyzepsbt", // avoid signed integer overflow in CFeeRate::GetFee(unsigned long) (https://github.com/bitcoin/bitcoin/issues/20607) + "dumptxoutset", // avoid writing to disk +#ifdef ENABLE_WALLET + "dumpwallet", // avoid writing to disk +#endif + "echoipc", // avoid assertion failure (Assertion `"EnsureAnyNodeContext(request.context).init" && check' failed.) + "generatetoaddress", // avoid timeout + "gettxoutproof", // avoid slow execution +#ifdef ENABLE_WALLET + "importwallet", // avoid reading from disk + "loadwallet", // avoid reading from disk +#endif + "mockscheduler", // avoid assertion failure (Assertion `delta_seconds.count() > 0 && delta_seconds < std::chrono::hours{1}' failed.) + "prioritisetransaction", // avoid signed integer overflow in CTxMemPool::PrioritiseTransaction(uint256 const&, long const&) (https://github.com/bitcoin/bitcoin/issues/20626) + "setban", // avoid DNS lookups + "stop", // avoid shutdown state +}; + +// RPC commands which are safe for fuzzing. +const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ + "clearbanned", + "combinepsbt", + "combinerawtransaction", + "converttopsbt", + "createmultisig", + "createpsbt", + "createrawtransaction", + "decodepsbt", + "decoderawtransaction", + "decodescript", + "deriveaddresses", + "disconnectnode", + "echo", + "echojson", + "estimaterawfee", + "estimatesmartfee", + "finalizepsbt", + "generate", + "generateblock", + "generatetodescriptor", + "getaddednodeinfo", + "getbestblockhash", + "getblock", + "getblockchaininfo", + "getblockcount", + "getblockfilter", + "getblockhash", + "getblockheader", + "getblockstats", + "getblocktemplate", + "getchaintips", + "getchaintxstats", + "getconnectioncount", + "getdescriptorinfo", + "getdifficulty", + "getindexinfo", + "getmemoryinfo", + "getmempoolancestors", + "getmempooldescendants", + "getmempoolentry", + "getmempoolinfo", + "getmininginfo", + "getnettotals", + "getnetworkhashps", + "getnetworkinfo", + "getnodeaddresses", + "getpeerinfo", + "getrawmempool", + "getrawtransaction", + "getrpcinfo", + "gettxout", + "gettxoutsetinfo", + "help", + "invalidateblock", + "joinpsbts", + "listbanned", + "logging", + "ping", + "preciousblock", + "pruneblockchain", + "reconsiderblock", + "savemempool", + "scantxoutset", + "sendrawtransaction", + "setmocktime", + "setnetworkactive", + "signmessagewithprivkey", + "signrawtransactionwithkey", + "submitblock", + "submitheader", + "syncwithvalidationinterfacequeue", + "testmempoolaccept", + "uptime", + "utxoupdatepsbt", + "validateaddress", + "verifychain", + "verifymessage", + "verifytxoutproof", + "waitforblock", + "waitforblockheight", + "waitfornewblock", +}; + +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + const size_t max_string_length = 4096; + std::string r; + CallOneOf( + fuzzed_data_provider, + [&] { + // string argument + r = fuzzed_data_provider.ConsumeRandomLengthString(max_string_length); + }, + [&] { + // base64 argument + r = EncodeBase64(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // hex argument + r = HexStr(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // bool argument + r = fuzzed_data_provider.ConsumeBool() ? "true" : "false"; + }, + [&] { + // range argument + r = "[" + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "," + ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()) + "]"; + }, + [&] { + // integral argument (int64_t) + r = ToString(fuzzed_data_provider.ConsumeIntegral<int64_t>()); + }, + [&] { + // integral argument (uint64_t) + r = ToString(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + }, + [&] { + // floating point argument + r = strprintf("%f", fuzzed_data_provider.ConsumeFloatingPoint<double>()); + }, + [&] { + // tx destination argument + r = EncodeDestination(ConsumeTxDestination(fuzzed_data_provider)); + }, + [&] { + // uint160 argument + r = ConsumeUInt160(fuzzed_data_provider).ToString(); + }, + [&] { + // uint256 argument + r = ConsumeUInt256(fuzzed_data_provider).ToString(); + }, + [&] { + // base32 argument + r = EncodeBase32(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length)); + }, + [&] { + // base58 argument + r = EncodeBase58(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length))); + }, + [&] { + // base58 argument with checksum + r = EncodeBase58Check(MakeUCharSpan(fuzzed_data_provider.ConsumeRandomLengthString(max_string_length))); + }, + [&] { + // hex encoded block + std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + if (!opt_block) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_block; + r = HexStr(data_stream); + }, + [&] { + // hex encoded block header + std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); + if (!opt_block_header) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_block_header; + r = HexStr(data_stream); + }, + [&] { + // hex encoded tx + std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (!opt_tx) { + return; + } + CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; + data_stream << *opt_tx; + r = HexStr(data_stream); + }, + [&] { + // base64 encoded psbt + std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); + if (!opt_psbt) { + return; + } + CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + data_stream << *opt_psbt; + r = EncodeBase64({data_stream.begin(), data_stream.end()}); + }, + [&] { + // base58 encoded key + const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32); + CKey key; + key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + if (!key.IsValid()) { + return; + } + r = EncodeSecret(key); + }, + [&] { + // hex encoded pubkey + const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(32); + CKey key; + key.Set(random_bytes.begin(), random_bytes.end(), fuzzed_data_provider.ConsumeBool()); + if (!key.IsValid()) { + return; + } + r = HexStr(key.GetPubKey()); + }); + return r; +} + +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + std::vector<std::string> scalar_arguments; + while (fuzzed_data_provider.ConsumeBool()) { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + } + return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; +} + +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +{ + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); +} + +RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() +{ + static const auto setup = MakeNoLogFileContext<RPCFuzzTestingSetup>(); + SetRPCWarmupFinished(); + return setup.get(); +} +}; // namespace + +void initialize_rpc() +{ + rpc_testing_setup = InitializeRPCFuzzTestingSetup(); + const std::vector<std::string> supported_rpc_commands = rpc_testing_setup->GetRPCCommands(); + for (const std::string& rpc_command : supported_rpc_commands) { + const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end(); + const bool not_safe_for_fuzzing = std::find(RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_NOT_SAFE_FOR_FUZZING.end(); + if (!(safe_for_fuzzing || not_safe_for_fuzzing)) { + std::cerr << "Error: RPC command \"" << rpc_command << "\" not found in RPC_COMMANDS_SAFE_FOR_FUZZING or RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + if (safe_for_fuzzing && not_safe_for_fuzzing) { + std::cerr << "Error: RPC command \"" << rpc_command << "\" found in *both* RPC_COMMANDS_SAFE_FOR_FUZZING and RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + for (const std::string& rpc_command : RPC_COMMANDS_SAFE_FOR_FUZZING) { + const bool supported_rpc_command = std::find(supported_rpc_commands.begin(), supported_rpc_commands.end(), rpc_command) != supported_rpc_commands.end(); + if (!supported_rpc_command) { + std::cerr << "Error: Unknown RPC command \"" << rpc_command << "\" found in RPC_COMMANDS_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + for (const std::string& rpc_command : RPC_COMMANDS_NOT_SAFE_FOR_FUZZING) { + const bool supported_rpc_command = std::find(supported_rpc_commands.begin(), supported_rpc_commands.end(), rpc_command) != supported_rpc_commands.end(); + if (!supported_rpc_command) { + std::cerr << "Error: Unknown RPC command \"" << rpc_command << "\" found in RPC_COMMANDS_NOT_SAFE_FOR_FUZZING. Please update " << __FILE__ << ".\n"; + std::terminate(); + } + } + const char* limit_to_rpc_command_env = std::getenv("LIMIT_TO_RPC_COMMAND"); + if (limit_to_rpc_command_env != nullptr) { + g_limit_to_rpc_command = std::string{limit_to_rpc_command_env}; + } +} + +FUZZ_TARGET_INIT(rpc, initialize_rpc) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + SetMockTime(ConsumeTime(fuzzed_data_provider)); + const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); + if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { + return; + } + const bool safe_for_fuzzing = std::find(RPC_COMMANDS_SAFE_FOR_FUZZING.begin(), RPC_COMMANDS_SAFE_FOR_FUZZING.end(), rpc_command) != RPC_COMMANDS_SAFE_FOR_FUZZING.end(); + if (!safe_for_fuzzing) { + return; + } + std::vector<std::string> arguments; + while (fuzzed_data_provider.ConsumeBool()) { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + } + try { + rpc_testing_setup->CallRPC(rpc_command, arguments); + } catch (const UniValue&) { + } catch (const std::runtime_error&) { + } +} diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index e87ae5b04b..b87bcf2ef5 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -43,7 +43,7 @@ FUZZ_TARGET_INIT(script, initialize_script) if (!script_opt) return; const CScript script{*script_opt}; - std::vector<unsigned char> compressed; + CompressedScript compressed; if (CompressScript(script, compressed)) { const unsigned int size = compressed[0]; compressed.erase(compressed.begin()); @@ -55,22 +55,45 @@ FUZZ_TARGET_INIT(script, initialize_script) } CTxDestination address; - (void)ExtractDestination(script, address); - TxoutType type_ret; std::vector<CTxDestination> addresses; int required_ret; - (void)ExtractDestinations(script, type_ret, addresses, required_ret); - - const FlatSigningProvider signing_provider; - (void)InferDescriptor(script, signing_provider); - - (void)IsSegWitOutput(signing_provider, script); - - (void)IsSolvable(signing_provider, script); + bool extract_destinations_ret = ExtractDestinations(script, type_ret, addresses, required_ret); + bool extract_destination_ret = ExtractDestination(script, address); + if (!extract_destinations_ret) { + assert(!extract_destination_ret); + if (type_ret == TxoutType::MULTISIG) { + assert(addresses.empty() && required_ret == 0); + } else { + assert(type_ret == TxoutType::PUBKEY || + type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA); + } + } else { + assert(required_ret >= 1 && required_ret <= 16); + assert((unsigned long)required_ret == addresses.size()); + assert(type_ret == TxoutType::MULTISIG || required_ret == 1); + } + if (type_ret == TxoutType::NONSTANDARD || type_ret == TxoutType::NULL_DATA) { + assert(!extract_destinations_ret); + } + if (!extract_destination_ret) { + assert(type_ret == TxoutType::PUBKEY || + type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA || + type_ret == TxoutType::MULTISIG); + } else { + assert(address == addresses[0]); + } + if (type_ret == TxoutType::NONSTANDARD || + type_ret == TxoutType::NULL_DATA || + type_ret == TxoutType::MULTISIG) { + assert(!extract_destination_ret); + } TxoutType which_type; bool is_standard_ret = IsStandard(script, which_type); + assert(type_ret == which_type); if (!is_standard_ret) { assert(which_type == TxoutType::NONSTANDARD || which_type == TxoutType::NULL_DATA || @@ -87,6 +110,11 @@ FUZZ_TARGET_INIT(script, initialize_script) which_type == TxoutType::NONSTANDARD); } + const FlatSigningProvider signing_provider; + (void)InferDescriptor(script, signing_provider); + (void)IsSegWitOutput(signing_provider, script); + (void)IsSolvable(signing_provider, script); + (void)RecursiveDynamicUsage(script); std::vector<std::vector<unsigned char>> solutions; @@ -115,10 +143,12 @@ FUZZ_TARGET_INIT(script, initialize_script) { const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + CompressedScript compressed_script; + compressed_script.assign(bytes.begin(), bytes.end()); // DecompressScript(..., ..., bytes) is not guaranteed to be defined if the bytes vector is too short - if (bytes.size() >= 32) { + if (compressed_script.size() >= 32) { CScript decompressed_script; - DecompressScript(decompressed_script, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), bytes); + DecompressScript(decompressed_script, fuzzed_data_provider.ConsumeIntegral<unsigned int>(), compressed_script); } } diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index aa911cdeda..1278dc87d4 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -41,6 +41,10 @@ FUZZ_TARGET_INIT(script_flags, initialize_script_flags) for (unsigned i = 0; i < tx.vin.size(); ++i) { CTxOut prevout; ds >> prevout; + if (!MoneyRange(prevout.nValue)) { + // prevouts should be consensus-valid + prevout.nValue = 1; + } spent_outputs.push_back(prevout); } PrecomputedTransactionData txdata; diff --git a/src/validation.cpp b/src/validation.cpp index 09b7bb4398..bf774a0791 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4101,38 +4101,46 @@ CVerifyDB::~CVerifyDB() uiInterface.ShowProgress("", 100, false); } -bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_chainstate, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) +bool CVerifyDB::VerifyDB( + CChainState& chainstate, + const CChainParams& chainparams, + CCoinsView& coinsview, + int nCheckLevel, int nCheckDepth) { AssertLockHeld(cs_main); - assert(std::addressof(::ChainstateActive()) == std::addressof(active_chainstate)); - if (active_chainstate.m_chain.Tip() == nullptr || active_chainstate.m_chain.Tip()->pprev == nullptr) + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + if (chainstate.m_chain.Tip() == nullptr || chainstate.m_chain.Tip()->pprev == nullptr) return true; // Verify blocks in the best chain - if (nCheckDepth <= 0 || nCheckDepth > active_chainstate.m_chain.Height()) - nCheckDepth = active_chainstate.m_chain.Height(); + if (nCheckDepth <= 0 || nCheckDepth > chainstate.m_chain.Height()) + nCheckDepth = chainstate.m_chain.Height(); nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); - CCoinsViewCache coins(coinsview); + CCoinsViewCache coins(&coinsview); CBlockIndex* pindex; CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; BlockValidationState state; int reportDone = 0; LogPrintf("[0%%]..."); /* Continued */ - for (pindex = active_chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { - const int percentageDone = std::max(1, std::min(99, (int)(((double)(active_chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + + bool is_snapshot_cs = !chainstate.m_from_snapshot_blockhash.IsNull(); + + for (pindex = chainstate.m_chain.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { + const int percentageDone = std::max(1, std::min(99, (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone/10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); - if (pindex->nHeight <= active_chainstate.m_chain.Height()-nCheckDepth) + if (pindex->nHeight <= chainstate.m_chain.Height()-nCheckDepth) break; - if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, only go back as far as we have data. + if ((fPruneMode || is_snapshot_cs) && !(pindex->nStatus & BLOCK_HAVE_DATA)) { + // If pruning or running under an assumeutxo snapshot, only go + // back as far as we have data. LogPrintf("VerifyDB(): block verification stopping at height %d (pruning, no data)\n", pindex->nHeight); break; } @@ -4154,9 +4162,11 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_ch } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + active_chainstate.CoinsTip().DynamicMemoryUsage()) <= active_chainstate.m_coinstip_cache_size_bytes) { + size_t curr_coins_usage = coins.DynamicMemoryUsage() + chainstate.CoinsTip().DynamicMemoryUsage(); + + if (nCheckLevel >= 3 && curr_coins_usage <= chainstate.m_coinstip_cache_size_bytes) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); - DisconnectResult res = active_chainstate.DisconnectBlock(block, pindex, coins); + DisconnectResult res = chainstate.DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4170,26 +4180,26 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CChainState& active_ch if (ShutdownRequested()) return true; } if (pindexFailure) - return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", active_chainstate.m_chain.Height() - pindexFailure->nHeight + 1, nGoodTransactions); + return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainstate.m_chain.Height() - pindexFailure->nHeight + 1, nGoodTransactions); // store block count as we move pindex at check level >= 4 - int block_count = active_chainstate.m_chain.Height() - pindex->nHeight; + int block_count = chainstate.m_chain.Height() - pindex->nHeight; // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { - while (pindex != active_chainstate.m_chain.Tip()) { - const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(active_chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); + while (pindex != chainstate.m_chain.Tip()) { + const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(chainstate.m_chain.Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); if (reportDone < percentageDone/10) { // report every 10% step LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); - pindex = active_chainstate.m_chain.Next(pindex); + pindex = chainstate.m_chain.Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!active_chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) + if (!chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); if (ShutdownRequested()) return true; } @@ -5165,14 +5175,14 @@ bool ChainstateManager::PopulateAndValidateSnapshot( return false; } - CCoinsStats stats; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ }; // As above, okay to immediately release cs_main here since no other context knows // about the snapshot_chainstate. CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); - if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, CoinStatsHashType::HASH_SERIALIZED, breakpoint_fnc)) { + if (!GetUTXOStats(snapshot_coinsdb, WITH_LOCK(::cs_main, return std::ref(m_blockman)), stats, breakpoint_fnc)) { LogPrintf("[snapshot] failed to generate coins stats\n"); return false; } diff --git a/src/validation.h b/src/validation.h index 9ae2838ead..63a8bdc096 100644 --- a/src/validation.h +++ b/src/validation.h @@ -79,6 +79,7 @@ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; @@ -195,14 +196,14 @@ struct MempoolAcceptResult { VALID, //!> Fully validated, valid. INVALID, //!> Invalid. }; - ResultType m_result_type; - TxValidationState m_state; + const ResultType m_result_type; + const TxValidationState m_state; // The following fields are only present when m_result_type = ResultType::VALID /** Mempool transactions replaced by the tx per BIP 125 rules. */ - std::optional<std::list<CTransactionRef>> m_replaced_transactions; - /** Raw base fees. */ - std::optional<CAmount> m_base_fees; + const std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Raw base fees in satoshis. */ + const std::optional<CAmount> m_base_fees; /** Constructor for failure case */ explicit MempoolAcceptResult(TxValidationState state) @@ -212,7 +213,7 @@ struct MempoolAcceptResult { /** Constructor for success case */ explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) - : m_result_type(ResultType::VALID), m_state(TxValidationState{}), + : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} }; @@ -329,7 +330,12 @@ class CVerifyDB { public: CVerifyDB(); ~CVerifyDB(); - bool VerifyDB(const CChainParams& chainparams, CChainState& active_chainstate, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool VerifyDB( + CChainState& chainstate, + const CChainParams& chainparams, + CCoinsView& coinsview, + int nCheckLevel, + int nCheckDepth) EXCLUSIVE_LOCKS_REQUIRED(cs_main); }; enum DisconnectResult @@ -913,6 +919,8 @@ public: return m_blockman.m_block_index; } + //! @returns true if a snapshot-based chainstate is in use. Also implies + //! that a background validation chainstate is also in use. bool IsSnapshotActive() const; std::optional<uint256> SnapshotBlockhash() const; diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index f0e1addaf1..5c1b36be6e 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -15,6 +15,7 @@ static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees static const CAmount MIN_FINAL_CHANGE = MIN_CHANGE/2; +/** A UTXO under consideration for use in funding a new transaction. */ class CInputCoin { public: CInputCoin(const CTransactionRef& tx, unsigned int i) @@ -56,31 +57,58 @@ public: } }; +/** Parameters for filtering which OutputGroups we may use in coin selection. + * We start by being very selective and requiring multiple confirmations and + * then get more permissive if we cannot fund the transaction. */ struct CoinEligibilityFilter { + /** Minimum number of confirmations for outputs that we sent to ourselves. + * We may use unconfirmed UTXOs sent from ourselves, e.g. change outputs. */ const int conf_mine; + /** Minimum number of confirmations for outputs received from a different + * wallet. We never spend unconfirmed foreign outputs as we cannot rely on these funds yet. */ const int conf_theirs; + /** Maximum number of unconfirmed ancestors aggregated across all UTXOs in an OutputGroup. */ const uint64_t max_ancestors; + /** Maximum number of descendants that a single UTXO in the OutputGroup may have. */ const uint64_t max_descendants; - const bool m_include_partial_groups{false}; //! Include partial destination groups when avoid_reuse and there are full groups + /** When avoid_reuse=true and there are full groups (OUTPUT_GROUP_MAX_ENTRIES), whether or not to use any partial groups.*/ + const bool m_include_partial_groups{false}; CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_ancestors) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants, bool include_partial) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants), m_include_partial_groups(include_partial) {} }; +/** A group of UTXOs paid to the same output script. */ struct OutputGroup { + /** The list of UTXOs contained in this output group. */ std::vector<CInputCoin> m_outputs; + /** Whether the UTXOs were sent by the wallet to itself. This is relevant because we may want at + * least a certain number of confirmations on UTXOs received from outside wallets while trusting + * our own UTXOs more. */ bool m_from_me{true}; + /** The total value of the UTXOs in sum. */ CAmount m_value{0}; + /** The minimum number of confirmations the UTXOs in the group have. Unconfirmed is 0. */ int m_depth{999}; + /** The aggregated count of unconfirmed ancestors of all UTXOs in this + * group. Not deduplicated and may overestimate when ancestors are shared. */ size_t m_ancestors{0}; + /** The maximum count of descendants of a single UTXO in this output group. */ size_t m_descendants{0}; + /** The value of the UTXOs after deducting the cost of spending them at the effective feerate. */ CAmount effective_value{0}; + /** The fee to spend these UTXOs at the effective feerate. */ CAmount fee{0}; + /** The target feerate of the transaction we're trying to build. */ CFeeRate m_effective_feerate{0}; + /** The fee to spend these UTXOs at the long term feerate. */ CAmount long_term_fee{0}; + /** The feerate for spending a created change output eventually (i.e. not urgently, and thus at + * a lower feerate). Calculated using long term fee estimate. This is used to decide whether + * it could be economical to create a change output. */ CFeeRate m_long_term_feerate{0}; OutputGroup() {} diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 332e7b1397..f0aaee7e4e 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2478,7 +2478,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm } } - // remove preset inputs from vCoins + // remove preset inputs from vCoins so that Coin Selection doesn't pick them. for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coin_control.HasSelected();) { if (setPresetCoins.count(it->GetInputCoin())) @@ -2490,9 +2490,9 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm unsigned int limit_ancestor_count = 0; unsigned int limit_descendant_count = 0; chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); - size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); - size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); - bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + const size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); + const size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); + const bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); // form groups from remaining coins; note that preset coins will not // automatically have their associated (same address) coins included @@ -2502,16 +2502,53 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - bool res = value_to_select <= 0 || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); - // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset + // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the + // transaction at a target feerate. If an attempt fails, more attempts may be made using a more + // permissive CoinEligibilityFilter. + const bool res = [&] { + // Pre-selected inputs already cover the target amount. + if (value_to_select <= 0) return true; + + // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six + // confirmations on outputs received from other wallets and only spend confirmed change. + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) return true; + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) return true; + + // Fall back to using zero confirmation change (but with as few ancestors in the mempool as + // possible) if we cannot fund the transaction otherwise. We never spend unconfirmed + // outputs received from other wallets. + if (m_spend_zero_conf_change) { + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) return true; + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + // If partial groups are allowed, relax the requirement of spending OutputGroups (groups + // of UTXOs sent to the same address, which are obviously controlled by a single wallet) + // in their entirety. + if (SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + // Try with unlimited ancestors/descendants. The transaction will still need to meet + // mempool ancestor/descendant policy to be accepted to mempool and broadcasted, but + // OutputGroups use heuristics that may overestimate ancestor/descendant counts. + if (!fRejectLongChains && SelectCoinsMinConf(value_to_select, + CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), + vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) { + return true; + } + } + // Coin Selection failed. + return false; + }(); + + // SelectCoinsMinConf clears setCoinsRet, so add the preset inputs from coin_control to the coinset util::insert(setCoinsRet, setPresetCoins); // add preset inputs to the total value selected diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c4acef8705..03adca7a89 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -368,7 +368,7 @@ public: CTransactionRef tx; - /* New transactions start as UNCONFIRMED. At BlockConnected, + /** New transactions start as UNCONFIRMED. At BlockConnected, * they will transition to CONFIRMED. In case of reorg, at BlockDisconnected, * they roll back to UNCONFIRMED. If we detect a conflicting transaction at * block connection, we update conflicted tx and its dependencies as CONFLICTED. @@ -383,7 +383,7 @@ public: ABANDONED }; - /* Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} + /** Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} * at which tx has been confirmed. All three are set to 0 if tx is unconfirmed or abandoned. * Meaning of these fields changes with CONFLICTED state where they instead point to block hash * and block height of the deepest conflicting tx. @@ -481,7 +481,7 @@ public: CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const; CAmount GetChange() const; - // Get the marginal bytes if spending the specified output from this transaction + /** Get the marginal bytes if spending the specified output from this transaction */ int GetSpendSize(unsigned int out, bool use_max_sig = false) const { return CalculateMaximumSignedInputSize(tx->vout[out], pwallet, use_max_sig); @@ -495,7 +495,7 @@ public: return (GetDebit(filter) > 0); } - // True if only scriptSigs are different + /** True if only scriptSigs are different */ bool IsEquivalentTo(const CWalletTx& tx) const; bool InMempool() const; @@ -503,7 +503,7 @@ public: int64_t GetTxTime() const; - // Pass this transaction to node for mempool insertion and relay to peers if flag set to true + /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct @@ -564,7 +564,15 @@ class COutput { public: const CWalletTx *tx; + + /** Index in tx->vout. */ int i; + + /** + * Depth in block chain. + * If > 0: the tx is on chain and has this many confirmations. + * If = 0: the tx is waiting confirmation. + * If < 0: a conflicting tx is on chain and has this many confirmations. */ int nDepth; /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ @@ -604,17 +612,30 @@ public: } }; +/** Parameters for one iteration of Coin Selection. */ struct CoinSelectionParams { + /** Toggles use of Branch and Bound instead of Knapsack solver. */ bool use_bnb = true; + /** Size of a change output in bytes, determined by the output type. */ size_t change_output_size = 0; + /** Size of the input to spend a change output in virtual bytes. */ size_t change_spend_size = 0; + /** The targeted feerate of the transaction being built. */ CFeeRate m_effective_feerate; + /** The feerate estimate used to estimate an upper bound on what should be sufficient to spend + * the change output sometime in the future. */ CFeeRate m_long_term_feerate; + /** If the cost to spend a change output at the discard feerate exceeds its value, drop it to fees. */ CFeeRate m_discard_feerate; + /** Size of the transaction before coin selection, consisting of the header and recipient + * output(s), excluding the inputs and change output(s). */ size_t tx_noinputs_size = 0; - //! Indicate that we are subtracting the fee from outputs + /** Indicate that we are subtracting the fee from outputs */ bool m_subtract_fee_outputs = false; + /** When true, always spend all (up to OUTPUT_GROUP_MAX_ENTRIES) or none of the outputs + * associated with the same address. This helps reduce privacy leaks resulting from address + * reuse. Dust outputs are not eligible to be added to output groups and thus not considered. */ bool m_avoid_partial_spends = false; CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_feerate, @@ -652,7 +673,10 @@ private: //! the current wallet version: clients below this version are not able to load the wallet int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; + /** The next scheduled rebroadcast of wallet transactions. */ int64_t nNextResend = 0; + /** Whether this wallet will submit newly created transactions to the node's mempool and + * prompt rebroadcasts (see ResendWalletTransactions()). */ bool fBroadcastTransactions = false; // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; @@ -682,10 +706,10 @@ private: */ bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /* Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ + /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); - /* Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ + /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -694,6 +718,7 @@ private: * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** WalletFlags set on this wallet. */ std::atomic<uint64_t> m_wallet_flags{0}; bool SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose); @@ -722,7 +747,7 @@ private: */ uint256 m_last_block_processed GUARDED_BY(cs_wallet); - /* Height of last block processed is used by wallet to know depth of transactions + /** Height of last block processed is used by wallet to know depth of transactions * without relying on Chain interface beyond asynchronous updates. For safety, we * initialize it to -1. Height is a pointer on node's tip and doesn't imply * that the wallet has scanned sequentially all blocks up to this one. @@ -739,7 +764,7 @@ private: bool CreateTransactionInternal(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, bool sign); public: - /* + /** * Main wallet lock. * This lock protects all the fields added by CWallet. */ @@ -753,8 +778,11 @@ public: /** * Select a set of coins such that nValueRet >= nTargetValue and at least - * all coins from coinControl are selected; Never select unconfirmed coins - * if they are not ours + * all coins from coin_control are selected; never select unconfirmed coins if they are not ours + * param@[out] setCoinsRet Populated with inputs including pre-selected inputs from + * coin_control and Coin Selection if successful. + * param@[out] nValueRet Total value of selected coins including pre-selected ones + * from coin_control and Coin Selection if successful. */ bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params, bool& bnb_used) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -788,6 +816,8 @@ public: /** Interface to assert chain access */ bool HaveChain() const { return m_chain ? true : false; } + /** Map from txid to CWalletTx for all transactions this wallet is + * interested in, including received and sent transactions. */ std::map<uint256, CWalletTx> mapWallet GUARDED_BY(cs_wallet); typedef std::multimap<int64_t, CWalletTx*> TxItems; @@ -799,6 +829,10 @@ public: std::map<CTxDestination, CAddressBookData> m_address_book GUARDED_BY(cs_wallet); const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Set of Coins owned by this wallet that we won't try to spend from. A + * Coin may be locked if it has already been used to fund a transaction + * that hasn't confirmed yet. We wouldn't consider the Coin spent already, + * but also shouldn't try to use it again. */ std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); /** Registered interfaces::Chain::Notifications handler. */ @@ -833,6 +867,11 @@ public: * small change; This method is stochastic for some inputs and upon * completion the coin set and corresponding actual target value is * assembled + * param@[in] coins Set of UTXOs to consider. These will be categorized into + * OutputGroups and filtered using eligibility_filter before + * selecting coins. + * param@[out] setCoinsRet Populated with the coins selected if successful. + * param@[out] nValueRet Used to return the total value of selected coins. */ bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; @@ -956,9 +995,9 @@ public: * calling CreateTransaction(); */ bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); - // Fetch the inputs and sign with SIGHASH_ALL. + /** Fetch the inputs and sign with SIGHASH_ALL. */ bool SignTransaction(CMutableTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - // Sign the tx given the input coins and sighash. + /** Sign the tx given the input coins and sighash. */ bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const; @@ -1015,6 +1054,8 @@ public: CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; + /** Allow Coin Selection to pick unconfirmed UTXOs that were sent from our own wallet if it + * cannot fund the transaction otherwise. */ bool m_spend_zero_conf_change{DEFAULT_SPEND_ZEROCONF_CHANGE}; bool m_signal_rbf{DEFAULT_WALLET_RBF}; bool m_allow_fallback_fee{true}; //!< will be false if -fallbackfee=0 @@ -1025,7 +1066,12 @@ public: * Override with -fallbackfee */ CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; + + /** If the cost to spend a change output at this feerate is greater than the value of the + * output itself, just drop it to fees. */ CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; + + /** The maximum fee amount we're willing to pay to prioritize partial spend avoidance. */ CAmount m_max_aps_fee{DEFAULT_MAX_AVOIDPARTIALSPEND_FEE}; //!< note: this is absolute fee, not fee rate OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; /** @@ -1333,10 +1379,10 @@ public: } }; -// Calculate the size of the transaction assuming all signatures are max size -// Use DummySignatureCreator, which inserts 71 byte signatures everywhere. -// NOTE: this requires that all inputs must be in mapWallet (eg the tx should -// be IsAllFromMe). +/** Calculate the size of the transaction assuming all signatures are max size +* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. +* NOTE: this requires that all inputs must be in mapWallet (eg the tx should +* be IsAllFromMe). */ std::pair<int64_t, int64_t> CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig = false) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); std::pair<int64_t, int64_t> CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, bool use_max_sig = false); diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py index d13d191b20..bada791fba 100755 --- a/test/functional/feature_blockfilterindex_prune.py +++ b/test/functional/feature_blockfilterindex_prune.py @@ -33,7 +33,7 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): self.log.info("prune some blocks") pruneheight = self.nodes[0].pruneblockchain(400) - assert_equal(pruneheight, 250) + assert_equal(pruneheight, 248) self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py new file mode 100755 index 0000000000..d3adde5cc5 --- /dev/null +++ b/test/functional/feature_coinstatsindex.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test coinstatsindex across nodes. + +Test that the values returned by gettxoutsetinfo are consistent +between a node running the coinstatsindex and a node without +the index. +""" + +from decimal import Decimal + +from test_framework.blocktools import ( + create_block, + create_coinbase, +) +from test_framework.messages import ( + COIN, + COutPoint, + CTransaction, + CTxIn, + CTxOut, + ToHex, +) +from test_framework.script import ( + CScript, + OP_FALSE, + OP_RETURN, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, + try_rpc, +) + +class CoinStatsIndexTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.supports_cli = False + self.extra_args = [ + [], + ["-coinstatsindex"] + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self._test_coin_stats_index() + self._test_use_index_option() + self._test_reorg_index() + self._test_index_rejects_hash_serialized() + + def block_sanity_check(self, block_info): + block_subsidy = 50 + assert_equal( + block_info['prevout_spent'] + block_subsidy, + block_info['new_outputs_ex_coinbase'] + block_info['coinbase'] + block_info['unspendable'] + ) + + def _test_coin_stats_index(self): + node = self.nodes[0] + index_node = self.nodes[1] + # Both none and muhash options allow the usage of the index + index_hash_options = ['none', 'muhash'] + + # Generate a normal transaction and mine it + node.generate(101) + address = self.nodes[0].get_deterministic_priv_key().address + node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) + node.generate(1) + + self.sync_blocks(timeout=120) + + self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo)) + res0 = node.gettxoutsetinfo('none') + + # The fields 'disk_size' and 'transactions' do not exist on the index + del res0['disk_size'], res0['transactions'] + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + res1 = index_node.gettxoutsetinfo(hash_option) + # The fields 'block_info' and 'total_unspendable_amount' only exist on the index + del res1['block_info'], res1['total_unspendable_amount'] + res1.pop('muhash', None) + + # Everything left should be the same + assert_equal(res1, res0) + + self.log.info("Test that gettxoutsetinfo() can get fetch data on specific heights with index") + + # Generate a new tip + node.generate(5) + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + # Fetch old stats by height + res2 = index_node.gettxoutsetinfo(hash_option, 102) + del res2['block_info'], res2['total_unspendable_amount'] + res2.pop('muhash', None) + assert_equal(res0, res2) + + # Fetch old stats by hash + res3 = index_node.gettxoutsetinfo(hash_option, res0['bestblock']) + del res3['block_info'], res3['total_unspendable_amount'] + res3.pop('muhash', None) + assert_equal(res0, res3) + + # It does not work without coinstatsindex + assert_raises_rpc_error(-8, "Querying specific block heights requires coinstatsindex", node.gettxoutsetinfo, hash_option, 102) + + self.log.info("Test gettxoutsetinfo() with index and verbose flag") + + for hash_option in index_hash_options: + # Genesis block is unspendable + res4 = index_node.gettxoutsetinfo(hash_option, 0) + assert_equal(res4['total_unspendable_amount'], 50) + assert_equal(res4['block_info'], { + 'unspendable': 50, + 'prevout_spent': 0, + 'new_outputs_ex_coinbase': 0, + 'coinbase': 0, + 'unspendables': { + 'genesis_block': 50, + 'bip30': 0, + 'scripts': 0, + 'unclaimed_rewards': 0 + } + }) + self.block_sanity_check(res4['block_info']) + + # Test an older block height that included a normal tx + res5 = index_node.gettxoutsetinfo(hash_option, 102) + assert_equal(res5['total_unspendable_amount'], 50) + assert_equal(res5['block_info'], { + 'unspendable': 0, + 'prevout_spent': 50, + 'new_outputs_ex_coinbase': Decimal('49.99995560'), + 'coinbase': Decimal('50.00004440'), + 'unspendables': { + 'genesis_block': 0, + 'bip30': 0, + 'scripts': 0, + 'unclaimed_rewards': 0 + } + }) + self.block_sanity_check(res5['block_info']) + + # Generate and send a normal tx with two outputs + tx1_inputs = [] + tx1_outputs = {self.nodes[0].getnewaddress(): 21, self.nodes[0].getnewaddress(): 42} + raw_tx1 = self.nodes[0].createrawtransaction(tx1_inputs, tx1_outputs) + funded_tx1 = self.nodes[0].fundrawtransaction(raw_tx1) + signed_tx1 = self.nodes[0].signrawtransactionwithwallet(funded_tx1['hex']) + tx1_txid = self.nodes[0].sendrawtransaction(signed_tx1['hex']) + + # Find the right position of the 21 BTC output + tx1_final = self.nodes[0].gettransaction(tx1_txid) + for output in tx1_final['details']: + if output['amount'] == Decimal('21.00000000') and output['category'] == 'receive': + n = output['vout'] + + # Generate and send another tx with an OP_RETURN output (which is unspendable) + tx2 = CTransaction() + tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b'')) + tx2.vout.append(CTxOut(int(20.99 * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) + tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx2))['hex'] + self.nodes[0].sendrawtransaction(tx2_hex) + + # Include both txs in a block + self.nodes[0].generate(1) + self.sync_all() + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + # Check all amounts were registered correctly + res6 = index_node.gettxoutsetinfo(hash_option, 108) + assert_equal(res6['total_unspendable_amount'], Decimal('70.98999999')) + assert_equal(res6['block_info'], { + 'unspendable': Decimal('20.98999999'), + 'prevout_spent': 111, + 'new_outputs_ex_coinbase': Decimal('89.99993620'), + 'coinbase': Decimal('50.01006381'), + 'unspendables': { + 'genesis_block': 0, + 'bip30': 0, + 'scripts': Decimal('20.98999999'), + 'unclaimed_rewards': 0 + } + }) + self.block_sanity_check(res6['block_info']) + + # Create a coinbase that does not claim full subsidy and also + # has two outputs + cb = create_coinbase(109, nValue=35) + cb.vout.append(CTxOut(5 * COIN, CScript([OP_FALSE]))) + cb.rehash() + + # Generate a block that includes previous coinbase + tip = self.nodes[0].getbestblockhash() + block_time = self.nodes[0].getblock(tip)['time'] + 1 + block = create_block(int(tip, 16), cb, block_time) + block.solve() + self.nodes[0].submitblock(ToHex(block)) + self.sync_all() + + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + for hash_option in index_hash_options: + res7 = index_node.gettxoutsetinfo(hash_option, 109) + assert_equal(res7['total_unspendable_amount'], Decimal('80.98999999')) + assert_equal(res7['block_info'], { + 'unspendable': 10, + 'prevout_spent': 0, + 'new_outputs_ex_coinbase': 0, + 'coinbase': 40, + 'unspendables': { + 'genesis_block': 0, + 'bip30': 0, + 'scripts': 0, + 'unclaimed_rewards': 10 + } + }) + self.block_sanity_check(res7['block_info']) + + self.log.info("Test that the index is robust across restarts") + + res8 = index_node.gettxoutsetinfo('muhash') + self.restart_node(1, extra_args=self.extra_args[1]) + res9 = index_node.gettxoutsetinfo('muhash') + assert_equal(res8, res9) + + index_node.generate(1) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res10 = index_node.gettxoutsetinfo('muhash') + assert(res8['txouts'] < res10['txouts']) + + def _test_use_index_option(self): + self.log.info("Test use_index option for nodes running the index") + + self.connect_nodes(0, 1) + self.nodes[0].waitforblockheight(110) + res = self.nodes[0].gettxoutsetinfo('muhash') + option_res = self.nodes[1].gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) + del res['disk_size'], option_res['disk_size'] + assert_equal(res, option_res) + + def _test_reorg_index(self): + self.log.info("Test that index can handle reorgs") + + # Generate two block, let the index catch up, then invalidate the blocks + index_node = self.nodes[1] + reorg_blocks = index_node.generatetoaddress(2, index_node.getnewaddress()) + reorg_block = reorg_blocks[1] + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res_invalid = index_node.gettxoutsetinfo('muhash') + index_node.invalidateblock(reorg_blocks[0]) + assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) + + # Add two new blocks + block = index_node.generate(2)[1] + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) + + # Test that the result of the reorged block is not returned for its old block height + res2 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) + assert_equal(res["bestblock"], block) + assert_equal(res["muhash"], res2["muhash"]) + assert(res["muhash"] != res_invalid["muhash"]) + + # Test that requesting reorged out block by hash is still returning correct results + res_invalid2 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=reorg_block) + assert_equal(res_invalid2["muhash"], res_invalid["muhash"]) + assert(res["muhash"] != res_invalid2["muhash"]) + + # Add another block, so we don't depend on reconsiderblock remembering which + # blocks were touched by invalidateblock + index_node.generate(1) + + # Ensure that removing and re-adding blocks yields consistent results + block = index_node.getblockhash(99) + index_node.invalidateblock(block) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + index_node.reconsiderblock(block) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + res3 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) + assert_equal(res2, res3) + + self.log.info("Test that a node aware of stale blocks syncs them as well") + node = self.nodes[0] + # Ensure the node is aware of a stale block prior to restart + node.getblock(reorg_block) + + self.restart_node(0, ["-coinstatsindex"]) + self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash')) + assert_raises_rpc_error(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash', reorg_block) + + def _test_index_rejects_hash_serialized(self): + self.log.info("Test that the rpc raises if the legacy hash is passed with the index") + + msg = "hash_serialized_2 hash type cannot be queried for a specific block" + assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111) + + for use_index in {True, False, None}: + assert_raises_rpc_error(-8, msg, self.nodes[1].gettxoutsetinfo, hash_type='hash_serialized_2', hash_or_height=111, use_index=use_index) + + +if __name__ == '__main__': + CoinStatsIndexTest().main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 2cf0ef2251..b5ce18a48b 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -151,7 +151,7 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['balance'], amounts[1]) self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balances") - cli_get_info = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli() + cli_get_info_keys = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli().keys() assert 'balance' not in cli_get_info_keys assert 'balances' not in cli_get_info_keys diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index a249a73315..b900aa0b9c 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -20,40 +20,41 @@ from test_framework.wallet import MiniWallet class MempoolSpendCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = True def run_test(self): wallet = MiniWallet(self.nodes[0]) - wallet.generate(200) - chain_height = self.nodes[0].getblockcount() - assert_equal(chain_height, 200) + # Invalidate two blocks, so that miniwallet has access to a coin that will mature in the next block + chain_height = 198 + self.nodes[0].invalidateblock(self.nodes[0].getblockhash(chain_height + 1)) + assert_equal(chain_height, self.nodes[0].getblockcount()) # Coinbase at height chain_height-100+1 ok in mempool, should # get mined. Coinbase at height chain_height-100+2 is # too immature to spend. - b = [self.nodes[0].getblockhash(n) for n in range(101, 103)] - coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] - utxo_101 = wallet.get_utxo(txid=coinbase_txids[0]) - utxo_102 = wallet.get_utxo(txid=coinbase_txids[1]) + wallet.scan_blocks(start=chain_height - 100 + 1, num=1) + utxo_mature = wallet.get_utxo() + wallet.scan_blocks(start=chain_height - 100 + 2, num=1) + utxo_immature = wallet.get_utxo() - spend_101_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_101)["txid"] + spend_mature_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_mature)["txid"] - # coinbase at height 102 should be too immature to spend + # other coinbase should be too immature to spend + immature_tx = wallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_immature, mempool_valid=False) assert_raises_rpc_error(-26, "bad-txns-premature-spend-of-coinbase", - lambda: wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)) + lambda: self.nodes[0].sendrawtransaction(immature_tx['hex'])) - # mempool should have just spend_101: - assert_equal(self.nodes[0].getrawmempool(), [spend_101_id]) + # mempool should have just the mature one + assert_equal(self.nodes[0].getrawmempool(), [spend_mature_id]) - # mine a block, spend_101 should get confirmed + # mine a block, mature one should get confirmed self.nodes[0].generate(1) assert_equal(set(self.nodes[0].getrawmempool()), set()) - # ... and now height 102 can be spent: - spend_102_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)["txid"] - assert_equal(self.nodes[0].getrawmempool(), [spend_102_id]) + # ... and now previously immature can be spent: + spend_new_id = self.nodes[0].sendrawtransaction(immature_tx['hex']) + assert_equal(self.nodes[0].getrawmempool(), [spend_new_id]) if __name__ == '__main__': diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index fdd02b7324..87297989ba 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -82,7 +82,7 @@ class AddrTest(BitcoinTestFramework): self.mocktime += 5 * 60 self.nodes[0].setmocktime(self.mocktime) for peer in receivers: - peer.sync_with_ping() + peer.sync_send_with_ping() def oversized_addr_test(self): self.log.info('Send an addr message that is too large') @@ -203,10 +203,7 @@ class AddrTest(BitcoinTestFramework): self.log.info('Check that we relay address messages') addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) msg = self.setup_addr_msg(2) - addr_source.send_and_ping(msg) - self.mocktime += 5 * 60 - self.nodes[0].setmocktime(self.mocktime) - full_outbound_peer.sync_with_ping() + self.send_addr_msg(addr_source, msg, [full_outbound_peer]) assert_equal(full_outbound_peer.num_ipv4_received, 2) self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6584efae79..445cea6186 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -89,11 +89,7 @@ class P2PBlocksOnly(BitcoinTestFramework): # Bump time forward to ensure nNextInvSend timer pops self.nodes[0].setmocktime(int(time.time()) + 60) - # Calling sync_with_ping twice requires that the node calls - # `ProcessMessage` twice, and thus ensures `SendMessages` must have - # been called at least once - conn.sync_with_ping() - conn.sync_with_ping() + conn.sync_send_with_ping() assert(int(txid, 16) not in conn.get_invs()) def check_p2p_tx_violation(self, index=1): diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 4bee33f825..359cfb9c34 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -174,8 +174,7 @@ class FilterTest(BitcoinTestFramework): filter_peer.merkleblock_received = False filter_peer.tx_received = False self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 90) - filter_peer.sync_with_ping() - filter_peer.sync_with_ping() + filter_peer.sync_send_with_ping() assert not filter_peer.merkleblock_received assert not filter_peer.tx_received diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 14a4afc2d7..9d32c1cb86 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -164,7 +164,7 @@ class TestP2PConn(P2PInterface): def on_wtxidrelay(self, message): self.last_wtxidrelay.append(message) - def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True, use_wtxid=False): + def announce_tx_and_wait_for_getdata(self, tx, success=True, use_wtxid=False): if success: # sanity check assert (self.wtxidrelay and use_wtxid) or (not self.wtxidrelay and not use_wtxid) @@ -178,11 +178,11 @@ class TestP2PConn(P2PInterface): if success: if use_wtxid: - self.wait_for_getdata([wtxid], timeout) + self.wait_for_getdata([wtxid]) else: - self.wait_for_getdata([tx.sha256], timeout) + self.wait_for_getdata([tx.sha256]) else: - time.sleep(timeout) + time.sleep(5) assert not self.last_message.get("getdata") def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): @@ -604,7 +604,7 @@ class SegWitTest(BitcoinTestFramework): # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. - self.test_node.announce_tx_and_wait_for_getdata(tx, timeout=2, success=False) + self.test_node.announce_tx_and_wait_for_getdata(tx, success=False) # Delivering this transaction with witness should fail (no matter who # its from) @@ -1461,7 +1461,7 @@ class SegWitTest(BitcoinTestFramework): self.std_node.announce_tx_and_wait_for_getdata(tx3) test_transaction_acceptance(self.nodes[1], self.std_node, tx3, with_witness=True, accepted=False, reason="bad-txns-nonstandard-inputs") # Now the node will no longer ask for getdata of this transaction when advertised by same txid - self.std_node.announce_tx_and_wait_for_getdata(tx3, timeout=5, success=False) + self.std_node.announce_tx_and_wait_for_getdata(tx3, success=False) # Spending a higher version witness output is not allowed by policy, # even with fRequireStandard=false. @@ -1966,9 +1966,10 @@ class SegWitTest(BitcoinTestFramework): # Restarting node 2 should result in a shutdown because the blockchain consists of # insufficiently validated blocks per segwit consensus rules. self.stop_node(2) - with self.nodes[2].assert_debug_log(expected_msgs=[ - f"Witness data for blocks after height {SEGWIT_HEIGHT} requires validation. Please restart with -reindex."], timeout=10): - self.nodes[2].start([f"-segwitheight={SEGWIT_HEIGHT}"]) + self.nodes[2].assert_start_raises_init_error( + extra_args=[f"-segwitheight={SEGWIT_HEIGHT}"], + expected_msg=f": Witness data for blocks after height {SEGWIT_HEIGHT} requires validation. Please restart with -reindex..\nPlease restart with -reindex or -reindex-chainstate to recover.", + ) # As directed, the user restarts the node with -reindex self.start_node(2, extra_args=["-reindex", f"-segwitheight={SEGWIT_HEIGHT}"]) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index ac53a280b4..00324347ed 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -275,7 +275,7 @@ class BlockchainTest(BitcoinTestFramework): assert 'muhash' in res6 assert(res['hash_serialized_2'] != res6['muhash']) - # muhash should not be included in gettxoutset unless requested. + # muhash should not be returned unless requested. for r in [res, res2, res3, res4, res5]: assert 'muhash' not in r diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index a80fa596cd..52c8fa883d 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -69,25 +69,22 @@ class RpcMiscTest(BitcoinTestFramework): assert_equal(node.getindexinfo(), {}) # Restart the node with indices and wait for them to sync - self.restart_node(0, ["-txindex", "-blockfilterindex"]) + self.restart_node(0, ["-txindex", "-blockfilterindex", "-coinstatsindex"]) self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) # Returns a list of all running indices by default + values = {"synced": True, "best_block_height": 200} assert_equal( node.getindexinfo(), { - "txindex": {"synced": True, "best_block_height": 200}, - "basic block filter index": {"synced": True, "best_block_height": 200} + "txindex": values, + "basic block filter index": values, + "coinstatsindex": values, } ) - # Specifying an index by name returns only the status of that index - assert_equal( - node.getindexinfo("txindex"), - { - "txindex": {"synced": True, "best_block_height": 200}, - } - ) + for i in {"txindex", "basic block filter index", "coinstatsindex"}: + assert_equal(node.getindexinfo(i), {i: values}) # Specifying an unknown index name returns an empty result assert_equal(node.getindexinfo("foo"), {}) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index e691b63df6..d08e025178 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -115,7 +115,7 @@ def script_BIP34_coinbase_height(height): return CScript([CScriptNum(height)]) -def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0): +def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValue=50): """Create a coinbase transaction. If pubkey is passed in, the coinbase output will be a P2PK output; @@ -126,10 +126,11 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0): coinbase = CTransaction() coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)) coinbaseoutput = CTxOut() - coinbaseoutput.nValue = 50 * COIN - halvings = int(height / 150) # regtest - coinbaseoutput.nValue >>= halvings - coinbaseoutput.nValue += fees + coinbaseoutput.nValue = nValue * COIN + if nValue == 50: + halvings = int(height / 150) # regtest + coinbaseoutput.nValue >>= halvings + coinbaseoutput.nValue += fees if pubkey is not None: coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG]) else: diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 05099f3339..cc80b543cd 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -539,8 +539,16 @@ class P2PInterface(P2PConnection): self.send_message(message) self.sync_with_ping(timeout=timeout) - # Sync up with the node + def sync_send_with_ping(self, timeout=60): + """Ensure SendMessages is called on this connection""" + # Calling sync_with_ping twice requires that the node calls + # `ProcessMessage` twice, and thus ensures `SendMessages` must have + # been called at least once + self.sync_with_ping() + self.sync_with_ping() + def sync_with_ping(self, timeout=60): + """Ensure ProcessMessages is called on this connection""" self.send_message(msg_ping(nonce=self.ping_counter)) def test_function(): diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 02eb10b5a4..a89a26caea 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -739,11 +739,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # block in the cache does not age too much (have an old tip age). # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). - gen_addresses = [k.address for k in TestNode.PRIV_KEYS] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] + gen_addresses = [k.address for k in TestNode.PRIV_KEYS][:3] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] + assert_equal(len(gen_addresses), 4) for i in range(8): cache_node.generatetoaddress( nblocks=25 if i != 7 else 24, - address=gen_addresses[i % 4], + address=gen_addresses[i % len(gen_addresses)], ) assert_equal(cache_node.getblockchaininfo()["blocks"], 199) @@ -757,7 +758,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): os.rmdir(cache_path('wallets')) # Remove empty wallets dir for entry in os.listdir(cache_path()): - if entry not in ['chainstate', 'blocks']: # Only keep chainstate and blocks folder + if entry not in ['chainstate', 'blocks', 'indexes']: # Only indexes, chainstate and blocks folders os.remove(cache_path(entry)) for i in range(self.num_nodes): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ce9c1bc024..c17c16f797 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -491,6 +491,7 @@ class TestNode(): self.start(extra_args, stdout=log_stdout, stderr=log_stderr, *args, **kwargs) ret = self.process.wait(timeout=self.rpc_timeout) self.log.debug(self._node_msg(f'bitcoind exited with status {ret} during initialization')) + assert ret != 0 # Exit code must indicate failure self.running = False self.process = None # Check stderr for expected message diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index a906a21dd0..59ef18635b 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -37,9 +37,13 @@ class MiniWallet: for i in range(start, start + num): block = self._test_node.getblock(blockhash=self._test_node.getblockhash(i), verbosity=2) for tx in block['tx']: - for out in tx['vout']: - if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): - self._utxos.append({'txid': tx['txid'], 'vout': out['n'], 'value': out['value']}) + self.scan_tx(tx) + + def scan_tx(self, tx): + """Scan the tx for self._scriptPubKey outputs and add them to self._utxos""" + for out in tx['vout']: + if out['scriptPubKey']['hex'] == self._scriptPubKey.hex(): + self._utxos.append({'txid': tx['txid'], 'vout': out['n'], 'value': out['value']}) def generate(self, num_blocks): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" @@ -69,6 +73,12 @@ class MiniWallet: def send_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None): """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + tx = self.create_self_transfer(fee_rate=fee_rate, from_node=from_node, utxo_to_spend=utxo_to_spend) + self.sendrawtransaction(from_node=from_node, tx_hex=tx['hex']) + return tx + + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None, mempool_valid=True): + """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" self._utxos = sorted(self._utxos, key=lambda k: k['value']) utxo_to_spend = utxo_to_spend or self._utxos.pop() # Pick the largest utxo (if none provided) and hope it covers the fee vsize = Decimal(96) @@ -84,8 +94,12 @@ class MiniWallet: tx_hex = tx.serialize().hex() tx_info = from_node.testmempoolaccept([tx_hex])[0] - self._utxos.append({'txid': tx_info['txid'], 'vout': 0, 'value': send_value}) - from_node.sendrawtransaction(tx_hex) - assert_equal(tx_info['vsize'], vsize) - assert_equal(tx_info['fees']['base'], fee) + assert_equal(mempool_valid, tx_info['allowed']) + if mempool_valid: + assert_equal(tx_info['vsize'], vsize) + assert_equal(tx_info['fees']['base'], fee) return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex} + + def sendrawtransaction(self, *, from_node, tx_hex): + from_node.sendrawtransaction(tx_hex) + self.scan_tx(from_node.decoderawtransaction(tx_hex)) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index bd58f2cd51..00527e78f1 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -281,6 +281,7 @@ BASE_SCRIPTS = [ 'rpc_scantxoutset.py', 'feature_logging.py', 'feature_anchors.py', + 'feature_coinstatsindex.py', 'p2p_node_network_limited.py', 'p2p_permissions.py', 'feature_blocksdir.py', diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index ad2333a808..354e14f361 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -14,6 +14,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "node/blockstorage -> validation -> node/blockstorage" "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" "index/base -> validation -> index/blockfilterindex -> index/base" + "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 27885b094e..3bcc554acb 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -96,6 +96,7 @@ implicit-unsigned-integer-truncation:crypto/ implicit-unsigned-integer-truncation:leveldb/ # std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage +shift-base:nanobench.h shift-base:*/include/c++/ shift-base:arith_uint256.cpp shift-base:crypto/ |