diff options
187 files changed, 3842 insertions, 1181 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 4b9b3839fa..1a972e1e4b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -24,11 +24,8 @@ filter_template: &FILTER_TEMPLATE base_template: &BASE_TEMPLATE << : *FILTER_TEMPLATE merge_base_script: - # Unconditionally install git (used in fingerprint_script) and set the - # default git author name (used in verify-commits.py) + # Unconditionally install git (used in fingerprint_script). - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - git fetch $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" - git checkout FETCH_HEAD # Use merged changes to detect silent merge conflicts @@ -117,14 +114,7 @@ task: QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' IgnoreWarnIntDirInTempDetected: 'true' merge_script: - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - # Windows filesystem loses the executable bit, and all of the executable - # files are considered "modified" now. It will break the following `git merge` - # command. The next two commands make git ignore this issue. - - git config core.filemode false - - git reset --hard - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } + - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } msvc_qt_built_cache: folder: "%QTBASEDIR%" reupload_on_changes: false @@ -181,7 +171,7 @@ task: - cd %CIRRUS_WORKING_DIR% - ccache --zero-stats --max-size=%CCACHE_SIZE% - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL% -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - ccache --show-stats check_script: - src\test_bitcoin.exe -l test_suite @@ -194,7 +184,7 @@ task: - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - netsh int ipv6 set dynamicport tcp start=1025 num=64511 # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash task: name: 'ARM [unit tests, no functional tests] [bullseye]' diff --git a/.tx/config b/.tx/config index 37c0bca0ab..20eff98d28 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[bitcoin.qt-translation-024x] +[o:bitcoin:p:bitcoin:r:qt-translation-024x] file_filter = src/qt/locale/bitcoin_<lang>.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/build_msvc/common.init.vcxproj.in b/build_msvc/common.init.vcxproj.in index 7f623c6296..24d5922182 100644 --- a/build_msvc/common.init.vcxproj.in +++ b/build_msvc/common.init.vcxproj.in @@ -88,7 +88,7 @@ <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions> - <DisableSpecificWarnings>4018;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> + <DisableSpecificWarnings>4018;4244;4267;4715;4805</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index ec572b4f2e..3a2540d549 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -54,6 +54,9 @@ <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj"> <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project> </ProjectReference> + <ProjectReference Include="..\libtest_util\libtest_util.vcxproj"> + <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project> + </ProjectReference> <ProjectReference Include="..\libleveldb\libleveldb.vcxproj"> <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> </ProjectReference> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 2b57411c3f..6f9a4709dd 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -7,7 +7,7 @@ export LC_ALL=C ${CI_RETRY_EXE} apt-get update -${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq +${CI_RETRY_EXE} apt-get install -y python3-pip curl git gawk jq ( # Temporary workaround for https://github.com/bitcoin/bitcoin/pull/26130#issuecomment-1260499544 # Can be removed once the underlying image is bumped to something that includes git2.34 or later @@ -15,8 +15,6 @@ ${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq ${CI_RETRY_EXE} apt-get update ${CI_RETRY_EXE} apt-get install -y --reinstall git ) -update-alternatives --install /usr/bin/clang-format clang-format "$(which clang-format-9 )" 100 -update-alternatives --install /usr/bin/clang-format-diff clang-format-diff "$(which clang-format-diff-9)" 100 ${CI_RETRY_EXE} pip3 install codespell==2.2.1 ${CI_RETRY_EXE} pip3 install flake8==4.0.1 diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 1f14dd079f..4506848740 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -31,6 +31,8 @@ if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; t git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root mapfile -t KEYS < contrib/verify-commits/trusted-keys + git config user.email "ci@ci.ci" + git config user.name "ci" ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys "${KEYS[@]}" && ./contrib/verify-commits/verify-commits.py; fi diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 8f1cc8af29..1ce3261f44 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -9,7 +9,8 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos export DOCKER_NAME_TAG=quay.io/centos/centos:stream8 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export PIP_PACKAGES="pyzmq" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 766424769d..76de87d955 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess export DOCKER_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3 python3-pip llvm clang g++-multilib" +export PACKAGES="cmake python3 llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index b706fb0906..b256ae21a5 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -10,12 +10,6 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then export LC_ALL=C fi -if [ "$CI_OS_NAME" == "macos" ]; then - sudo -H pip3 install --upgrade pip - # shellcheck disable=SC2086 - IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES -fi - # Create folders that are mounted into the docker mkdir -p "${CCACHE_DIR}" mkdir -p "${PREVIOUS_RELEASES_DIR}" @@ -78,9 +72,16 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then fi ${CI_RETRY_EXE} CI_EXEC apt-get update ${CI_RETRY_EXE} CI_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" - if [ -n "$PIP_PACKAGES" ]; then +fi + +if [ -n "$PIP_PACKAGES" ]; then + if [ "$CI_OS_NAME" == "macos" ]; then + sudo -H pip3 install --upgrade pip + # shellcheck disable=SC2086 + IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + else # shellcheck disable=SC2086 - ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES fi fi diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 5bdb392ba3..0ee80cf114 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -68,6 +68,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/util/string.cpp"\ " src/util/syserror.cpp"\ " src/util/url.cpp"\ + " src/zmq"\ " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi diff --git a/configure.ac b/configure.ac index 82cb469864..f2d621ec7f 100644 --- a/configure.ac +++ b/configure.ac @@ -588,8 +588,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$ARM_CRC_CXXFLAGS $CXXFLAGS" @@ -763,10 +763,6 @@ case $host in BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi - if test "$use_sqlite" != "no" && $BREW list --versions sqlite3 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" - fi - if $BREW list --versions qt@5 >/dev/null; then export PKG_CONFIG_PATH="$($BREW --prefix qt@5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index f8377cce33..2d96b863ef 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -1,3 +1,4 @@ +982A193E3CE0EED535E09023188CBB2648416AD5 0xB10C (0xb10c) 9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) 617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 8a3a14810d..f4778c1ecd 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -6,8 +6,15 @@ $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) +define $(package)_set_vars := +$(package)_config_opts := --with-external-capnp +$(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" +$(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" +$(package)_config_opts_android := --disable-shared +endef + define $(package)_config_cmds - $($(package)_autoconf) --with-external-capnp + $($(package)_autoconf) endef define $(package)_build_cmds diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 99f5b0a8db..7ad2529e47 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -3,17 +3,20 @@ $(package)_version=2.2.2 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=888fb0976ba61518276fe1eda988589c700a3f2a69d71089260d75562afd3687 -$(package)_patches=dont_leak_info.patch +$(package)_patches=dont_leak_info.patch respect_mingw_cflags.patch +# Next time this package is updated, ensure that _WIN32_WINNT is still properly set. +# See discussion in https://github.com/bitcoin/bitcoin/pull/25964. define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" $(package)_build_opts_darwin=LIBTOOL="$($(package)_libtool)" -$(package)_build_opts_mingw32=-f Makefile.mingw +$(package)_build_opts_mingw32=-f Makefile.mingw CFLAGS="$($(package)_cflags) -D_WIN32_WINNT=0x0601" $(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch + patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch && \ + patch -p1 < $($(package)_patch_dir)/respect_mingw_cflags.patch endef define $(package)_build_cmds diff --git a/depends/patches/miniupnpc/respect_mingw_cflags.patch b/depends/patches/miniupnpc/respect_mingw_cflags.patch new file mode 100644 index 0000000000..a44580ddab --- /dev/null +++ b/depends/patches/miniupnpc/respect_mingw_cflags.patch @@ -0,0 +1,23 @@ +commit fec515a7ac9991a0ee91068fda046b54b191155e +Author: fanquake <fanquake@gmail.com> +Date: Wed Jul 27 15:52:37 2022 +0100 + + build: respect CFLAGS in makefile.mingw + + Similar to the other Makefile. + + Cherry-pick of https://github.com/miniupnp/miniupnp/pull/619. + +diff --git a/Makefile.mingw b/Makefile.mingw +index 2bff7bd..88430d2 100644 +--- a/Makefile.mingw ++++ b/Makefile.mingw +@@ -19,7 +19,7 @@ else + RM = rm -f + endif + #CFLAGS = -Wall -g -DDEBUG -D_WIN32_WINNT=0X501 +-CFLAGS = -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 ++CFLAGS ?= -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 + LDLIBS = -lws2_32 -liphlpapi + # -lwsock32 + # -liphlpapi is needed for GetBestRoute() and GetIpAddrTable() diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 265b74ee9a..a704b969df 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -37,8 +37,8 @@ By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks -`GET /rest/block/<BLOCK-HASH>.<bin|hex|json>` -`GET /rest/block/notxdetails/<BLOCK-HASH>.<bin|hex|json>` +- `GET /rest/block/<BLOCK-HASH>.<bin|hex|json>` +- `GET /rest/block/notxdetails/<BLOCK-HASH>.<bin|hex|json>` Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. @@ -86,12 +86,23 @@ Returns various state info regarding block chain processing. Only supports JSON as output format. Refer to the `getblockchaininfo` RPC help for details. +#### Deployment info +`GET /rest/deploymentinfo.json` +`GET /rest/deploymentinfo/<BLOCKHASH>.json` + +Returns an object containing various state info regarding deployments of +consensus changes at the current chain tip, or at <BLOCKHASH> if provided. +Only supports JSON as output format. +Refer to the `getdeploymentinfo` RPC help for details. + #### Query UTXO set -`GET /rest/getutxos/<checkmempool>/<txid>-<n>/<txid>-<n>/.../<txid>-<n>.<bin|hex|json>` +- `GET /rest/getutxos/<TXID>-<N>/<TXID>-<N>/.../<TXID>-<N>.<bin|hex|json>` +- `GET /rest/getutxos/checkmempool/<TXID>-<N>/<TXID>-<N>/.../<TXID>-<N>.<bin|hex|json>` -The getutxo command allows querying of the UTXO set given a set of outpoints. -See BIP64 for input and output serialisation: -https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki +The getutxos endpoint allows querying the UTXO set, given a set of outpoints. +With the `/checkmempool/` option, the mempool is also taken into account. +See [BIP64](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) for +input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index c353c78ff8..ea51b1b87f 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -76,8 +76,9 @@ original chainstate remains in use as active. Once the snapshot chainstate is loaded and validated, it is promoted to active chainstate and a sync to tip begins. A new chainstate directory is created in the -datadir for the snapshot chainstate called -`chainstate_[SHA256 blockhash of snapshot base block]`. +datadir for the snapshot chainstate called `chainstate_snapshot`. When this directory +is present in the datadir, the snapshot chainstate will be detected and loaded as +active on node startup (via `DetectSnapshotChainstate()`). | | | | ---------- | ----------- | diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 75f8d60ba0..5e51f1e63f 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -83,7 +83,7 @@ class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold </td></tr></table> -- The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code still is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. +- The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. - *libbitcoin_consensus* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. diff --git a/doc/release-notes-22087.md b/doc/release-notes-22087.md new file mode 100644 index 0000000000..8d7fd242b2 --- /dev/null +++ b/doc/release-notes-22087.md @@ -0,0 +1,4 @@ +Updated settings +---------------- + +- Ports specified in `-port` and `-rpcport` options are now validated at startup. Values that previously worked and were considered valid can now result in errors. (#22087) diff --git a/doc/release-notes-25412.md b/doc/release-notes-25412.md new file mode 100644 index 0000000000..b11fe73d45 --- /dev/null +++ b/doc/release-notes-25412.md @@ -0,0 +1,5 @@ +New REST endpoint +----------------- + +- A new `/rest/deploymentinfo` endpoint has been added for fetching various + state info regarding deployments of consensus changes. (#25412) diff --git a/doc/release-notes/release-notes-0.20.2.md b/doc/release-notes/release-notes-0.20.2.md new file mode 100644 index 0000000000..ad001bc9c1 --- /dev/null +++ b/doc/release-notes/release-notes-0.20.2.md @@ -0,0 +1,165 @@ +0.20.2 Release Notes +==================== + +Bitcoin Core version 0.20.2 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.20.2/> + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Known Bugs +========== + +The process for generating the source code release ("tarball") has changed in an +effort to make it more complete, however, there are a few regressions in +this release: + +- The generated `configure` script is currently missing, and you will need to + install autotools and run `./autogen.sh` before you can run + `./configure`. This is the same as when checking out from git. + +- Instead of running `make` simply, you should instead run + `BITCOIN_GENBUILD_NO_GIT=1 make`. + +Notable changes +=============== + +Changes regarding misbehaving peers +----------------------------------- + +Peers that misbehave (e.g. send us invalid blocks) are now referred to as +discouraged nodes in log output, as they're not (and weren't) strictly banned: +incoming connections are still allowed from them, but they're preferred for +eviction. + +Furthermore, a few additional changes are introduced to how discouraged +addresses are treated: + +- Discouraging an address does not time out automatically after 24 hours + (or the `-bantime` setting). Depending on traffic from other peers, + discouragement may time out at an indeterminate time. + +- Discouragement is not persisted over restarts. + +- There is no method to list discouraged addresses. They are not returned by + the `listbanned` RPC. That RPC also no longer reports the `ban_reason` + field, as `"manually added"` is the only remaining option. + +- Discouragement cannot be removed with the `setban remove` RPC command. + If you need to remove a discouragement, you can remove all discouragements by + stop-starting your node. + +Notification changes +-------------------- + +`-walletnotify` notifications are now sent for wallet transactions that are +removed from the mempool because they conflict with a new block. These +notifications were sent previously before the v0.19 release, but had been +broken since that release (bug +[#18325](https://github.com/bitcoin/bitcoin/issues/18325)). + +PSBT changes +------------ + +PSBTs will contain both the non-witness utxo and the witness utxo for segwit +inputs in order to restore compatibility with wallet software that are now +requiring the full previous transaction for segwit inputs. The witness utxo +is still provided to maintain compatibility with software which relied on its +existence to determine whether an input was segwit. + +0.20.2 change log +================= + +### P2P protocol and network code + +- #19620 Add txids with non-standard inputs to reject filter (sdaftuar) +- #20146 Send post-verack handshake messages at most once (MarcoFalke) + +### Wallet + +- #19740 Simplify and fix CWallet::SignTransaction (achow101) + +### RPC and other APIs + +- #19836 Properly deserialize txs with witness before signing (MarcoFalke) +- #20731 Add missing description of vout in getrawtransaction help text (benthecarman) + +### Build system + +- #20142 build: set minimum required Boost to 1.48.0 (fanquake) +- #20298 use the new plistlib API (jonasschnelli) +- #20880 gitian: Use custom MacOS code signing tool (achow101) +- #22190 Use latest signapple commit (achow101) + +### Tests and QA + +- #19839 Set appveyor vm version to previous Visual Studio 2019 release. (sipsorcery) +- #19842 Update the vcpkg checkout commit ID in appveyor config. (sipsorcery) +- #20562 Test that a fully signed tx given to signrawtx is unchanged (achow101) + +### Miscellaneous + +- #19192 Extract net permissions doc (MarcoFalke) +- #19777 Correct description for getblockstats's txs field (shesek) +- #20080 Strip any trailing / in -datadir and -blocksdir paths (hebasto) +- #20082 fixes read buffer to use min rather than max (EthanHeilman) +- #20141 Avoid the use of abs64 in timedata (sipa) +- #20756 Add missing field (permissions) to the getpeerinfo help (amitiuttarwar) +- #20861 BIP 350: Implement Bech32m and use it for v1+ segwit addresses (sipa) +- #22124 Update translations after closing 0.20.x on Transifex (hebasto) +- #21471 fix bech32_encode calls in gen_key_io_test_vectors.py (sipa) +- #22837 mention bech32m/BIP350 in doc/descriptors.md (sipa) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Amiti Uttarwar +- Andrew Chow +- Ethan Heilman +- fanquake +- Hennadii Stepanov +- Jonas Schnelli +- MarcoFalke +- Nadav Ivgi +- Pieter Wuille +- Suhas Daftuar + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-0.21.2.md b/doc/release-notes/release-notes-0.21.2.md new file mode 100644 index 0000000000..3b33c48a26 --- /dev/null +++ b/doc/release-notes/release-notes-0.21.2.md @@ -0,0 +1,109 @@ +0.21.2 Release Notes +==================== + +Bitcoin Core version 0.21.2 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.21.2/> + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + + +0.21.2 change log +================= + +### P2P protocol and network code + +- #21644 use NetPermissions::HasFlag() in CConnman::Bind() (jonatack) +- #22569 Rate limit the processing of rumoured addresses (sipa) + +### Wallet + +- #21907 Do not iterate a directory if having an error while accessing it (hebasto) + +### RPC + +- #19361 Reset scantxoutset progress before inferring descriptors (prusnak) + +### Build System + +- #21932 depends: update Qt 5.9 source url (kittywhiskers) +- #22017 Update Windows code signing certificate (achow101) +- #22191 Use custom MacOS code signing tool (achow101) +- #22713 Fix build with Boost 1.77.0 (sizeofvoid) + +### Tests and QA + +- #20182 Build with --enable-werror by default, and document exceptions (hebasto) +- #20535 Fix intermittent feature_taproot issue (MarcoFalke) +- #21663 Fix macOS brew install command (hebasto) +- #22279 add missing ECCVerifyHandle to base_encode_decode (apoelstra) +- #22730 Run fuzzer task for the master branch only (hebasto) + +### GUI + +- #277 Do not use QClipboard::Selection on Windows and macOS. (hebasto) +- #280 Remove user input from URI error message (prayank23) +- #365 Draw "eye" sign at the beginning of watch-only addresses (hebasto) + +### Miscellaneous + +- #22002 Fix crash when parsing command line with -noincludeconf=0 (MarcoFalke) +- #22137 util: Properly handle -noincludeconf on command line (take 2) (MarcoFalke) + + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Andrew Poelstra +- fanquake +- Hennadii Stepanov +- Jon Atack +- Kittywhiskers Van Gogh +- Luke Dashjr +- MarcoFalke +- Pavol Rusnak +- Pieter Wuille +- prayank23 +- Rafael Sadowski +- W. J. van der Laan + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/src/Makefile.am b/src/Makefile.am index bc1c6a0d0f..70a0ca8915 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -133,6 +133,7 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ common/bloom.h \ + common/run_command.h \ compat/assumptions.h \ compat/byteswap.h \ compat/compat.h \ @@ -208,6 +209,7 @@ BITCOIN_CORE_H = \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ + node/txreconciliation.h \ node/utxo_snapshot.h \ node/validation_cache_args.h \ noui.h \ @@ -344,11 +346,7 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h - -# server: shared between bitcoind and bitcoin-qt -# Contains code accessing mempool and chain state that is meant to be separated -# from wallet and gui code (see node/README.md). Shared code should go in -# libbitcoin_common or libbitcoin_util libraries, instead. +# node # libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ @@ -395,6 +393,8 @@ libbitcoin_node_a_SOURCES = \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ + node/txreconciliation.cpp \ + node/utxo_snapshot.cpp \ node/validation_cache_args.cpp \ noui.cpp \ policy/fees.cpp \ @@ -437,7 +437,9 @@ endif if !ENABLE_WALLET libbitcoin_node_a_SOURCES += dummywallet.cpp endif +# +# zmq # if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -448,10 +450,9 @@ libbitcoin_zmq_a_SOURCES = \ zmq/zmqrpc.cpp \ zmq/zmqutil.cpp endif +# - -# wallet: shared between bitcoind and bitcoin-qt, but only linked -# when wallet enabled +# wallet # libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(BDB_CPPFLAGS) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ @@ -490,14 +491,17 @@ endif if USE_BDB libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp endif +# +# wallet tool # libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_tool_a_SOURCES = \ wallet/wallettool.cpp \ $(BITCOIN_CORE_H) +# -# crypto primitives library +# crypto # crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) # Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a @@ -577,8 +581,9 @@ crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp +# -# consensus: shared between all executables that validate any consensus rules. +# consensus # libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ @@ -614,9 +619,10 @@ libbitcoin_consensus_a_SOURCES = \ util/strencodings.cpp \ util/strencodings.h \ version.h +# -# common: shared between bitcoind, and bitcoin-qt and non-server tools -libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +# common # +libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ @@ -624,6 +630,7 @@ libbitcoin_common_a_SOURCES = \ chainparams.cpp \ coins.cpp \ common/bloom.cpp \ + common/run_command.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -653,8 +660,9 @@ libbitcoin_common_a_SOURCES = \ script/standard.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) +# -# util: shared between all executables. +# util # libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ @@ -702,8 +710,9 @@ libbitcoin_util_a_SOURCES = \ if USE_LIBEVENT libbitcoin_util_a_SOURCES += util/url.cpp endif +# -# cli: shared between bitcoin-cli and bitcoin-qt +# cli # libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ @@ -900,6 +909,7 @@ libbitcoinkernel_la_SOURCES = \ node/blockstorage.cpp \ node/chainstate.cpp \ node/interface_ui.cpp \ + node/utxo_snapshot.cpp \ policy/feerate.cpp \ policy/fees.cpp \ policy/packages.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 253f64d2c3..571a85e5c9 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -77,6 +77,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -147,6 +148,7 @@ BITCOIN_TESTS =\ test/transaction_tests.cpp \ test/txindex_tests.cpp \ test/txpackage_tests.cpp \ + test/txreconciliation_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ @@ -374,8 +376,8 @@ endif if TARGET_WINDOWS else if ENABLE_BENCH - @echo "Running bench/bench_bitcoin (one iteration sanity check)..." - $(BENCH_BINARY) --sanity-check > /dev/null + @echo "Running bench/bench_bitcoin (one iteration sanity check, only high priority)..." + $(BENCH_BINARY) -sanity-check -priority-level=high > /dev/null endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 11b5c12062..b35d713d57 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,12 +10,13 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ - test/fuzz/mempool_utils.h \ - test/fuzz/util.h + test/fuzz/util.h \ + test/fuzz/util/mempool.h libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ + test/fuzz/util/mempool.cpp \ $(TEST_FUZZ_H) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index ada789f1b0..d142572b27 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -5,20 +5,21 @@ LIBTEST_UTIL=libtest_util.a EXTRA_LIBRARIES += \ - $(LIBTEST_UTIL) + $(LIBTEST_UTIL) TEST_UTIL_H = \ - test/util/blockfilter.h \ - test/util/chainstate.h \ - test/util/logging.h \ - test/util/mining.h \ - test/util/net.h \ - test/util/script.h \ - test/util/setup_common.h \ - test/util/str.h \ - test/util/transaction_utils.h \ - test/util/validation.h \ - test/util/wallet.h + test/util/blockfilter.h \ + test/util/chainstate.h \ + test/util/logging.h \ + test/util/mining.h \ + test/util/net.h \ + test/util/script.h \ + test/util/setup_common.h \ + test/util/str.h \ + test/util/transaction_utils.h \ + test/util/txmempool.h \ + test/util/validation.h \ + test/util/wallet.h libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -31,6 +32,7 @@ libtest_util_a_SOURCES = \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ + test/util/txmempool.cpp \ test/util/validation.cpp \ test/util/wallet.cpp \ $(TEST_UTIL_H) diff --git a/src/bech32.cpp b/src/bech32.cpp index dce9b2e4cc..8e0025b8f4 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -241,7 +241,7 @@ constexpr std::array<uint32_t, 25> GenerateSyndromeConstants() { std::array<uint32_t, 25> SYNDROME_CONSTS{}; for (int k = 1; k < 6; ++k) { for (int shift = 0; shift < 5; ++shift) { - int16_t b = GF1024_LOG.at(1 << shift); + int16_t b = GF1024_LOG.at(size_t{1} << shift); int16_t c0 = GF1024_EXP.at((997*k + b) % 1023); int16_t c1 = GF1024_EXP.at((998*k + b) % 1023); int16_t c2 = GF1024_EXP.at((999*k + b) % 1023); diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index f14d1f89b6..019b133345 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -133,7 +133,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench) }); } -BENCHMARK(AddrManAdd); -BENCHMARK(AddrManSelect); -BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManAddThenGood); +BENCHMARK(AddrManAdd, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManSelect, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManGetAddr, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManAddThenGood, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 6f6b4e3bfa..3d08b7201b 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -50,6 +50,6 @@ static void Base58Decode(benchmark::Bench& bench) } -BENCHMARK(Base58Encode); -BENCHMARK(Base58CheckEncode); -BENCHMARK(Base58Decode); +BENCHMARK(Base58Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58CheckEncode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 0f89a8e2b5..1a166e7081 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -32,5 +32,5 @@ static void Bech32Decode(benchmark::Bench& bench) } -BENCHMARK(Bech32Encode); -BENCHMARK(Bech32Decode); +BENCHMARK(Bech32Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Bech32Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 26975bb59d..1a3a006286 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -6,6 +6,7 @@ #include <fs.h> #include <test/util/setup_common.h> +#include <util/string.h> #include <chrono> #include <fstream> @@ -41,18 +42,42 @@ void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& bench } // namespace -benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks() +namespace benchmark { + +// map a label to one or multiple priority levels +std::map<std::string, uint8_t> map_label_priority = { + {"high", PriorityLevel::HIGH}, + {"low", PriorityLevel::LOW}, + {"all", 0xff} +}; + +std::string ListPriorities() { - static std::map<std::string, BenchFunction> benchmarks_map; + using item_t = std::pair<std::string, uint8_t>; + auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; }; + std::set<item_t, decltype(sort_by_priority)> sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority); + return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; }); +} + +uint8_t StringToPriority(const std::string& str) +{ + auto it = map_label_priority.find(str); + if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str)); + return it->second; +} + +BenchRunner::BenchmarkMap& BenchRunner::benchmarks() +{ + static BenchmarkMap benchmarks_map; return benchmarks_map; } -benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) +BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level) { - benchmarks().insert(std::make_pair(name, func)); + benchmarks().insert(std::make_pair(name, std::make_pair(func, level))); } -void benchmark::BenchRunner::RunAll(const Args& args) +void BenchRunner::RunAll(const Args& args) { std::regex reFilter(args.regex_filter); std::smatch baseMatch; @@ -62,13 +87,19 @@ void benchmark::BenchRunner::RunAll(const Args& args) } std::vector<ankerl::nanobench::Result> benchmarkResults; - for (const auto& p : benchmarks()) { - if (!std::regex_match(p.first, baseMatch, reFilter)) { + for (const auto& [name, bench_func] : benchmarks()) { + const auto& [func, priority_level] = bench_func; + + if (!(priority_level & args.priority)) { + continue; + } + + if (!std::regex_match(name, baseMatch, reFilter)) { continue; } if (args.is_list_only) { - std::cout << p.first << std::endl; + std::cout << name << std::endl; continue; } @@ -76,7 +107,7 @@ void benchmark::BenchRunner::RunAll(const Args& args) if (args.sanity_check) { bench.epochs(1).epochIterations(1); } - bench.name(p.first); + bench.name(name); if (args.min_time > 0ms) { // convert to nanos before dividing to reduce rounding errors std::chrono::nanoseconds min_time_ns = args.min_time; @@ -84,11 +115,11 @@ void benchmark::BenchRunner::RunAll(const Args& args) } if (args.asymptote.empty()) { - p.second(bench); + func(bench); } else { for (auto n : args.asymptote) { bench.complexityN(n); - p.second(bench); + func(bench); } std::cout << bench.complexityBigO() << std::endl; } @@ -103,3 +134,5 @@ void benchmark::BenchRunner::RunAll(const Args& args) "{{/result}}"); GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json()); } + +} // namespace benchmark diff --git a/src/bench/bench.h b/src/bench/bench.h index 17535e4e81..63e1bf67e2 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -41,6 +41,16 @@ using ankerl::nanobench::Bench; typedef std::function<void(Bench&)> BenchFunction; +enum PriorityLevel : uint8_t +{ + LOW = 1 << 0, + HIGH = 1 << 2, +}; + +// List priority labels, comma-separated and sorted by increasing priority +std::string ListPriorities(); +uint8_t StringToPriority(const std::string& str); + struct Args { bool is_list_only; bool sanity_check; @@ -49,22 +59,24 @@ struct Args { fs::path output_csv; fs::path output_json; std::string regex_filter; + uint8_t priority; }; class BenchRunner { - typedef std::map<std::string, BenchFunction> BenchmarkMap; + // maps from "name" -> (function, priority_level) + typedef std::map<std::string, std::pair<BenchFunction, PriorityLevel>> BenchmarkMap; static BenchmarkMap& benchmarks(); public: - BenchRunner(std::string name, BenchFunction func); + BenchRunner(std::string name, BenchFunction func, PriorityLevel level); static void RunAll(const Args& args); }; } // namespace benchmark -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); -#define BENCHMARK(n) \ - benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo, priority_level); +#define BENCHMARK(n, priority_level) \ + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, priority_level); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 1bb4d34db9..1ac8db19fd 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -18,6 +18,8 @@ static const char* DEFAULT_BENCH_FILTER = ".*"; static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; +/** Priority level default value, run "all" priority levels */ +static const std::string DEFAULT_PRIORITY{"all"}; static void SetupBenchArgs(ArgsManager& argsman) { @@ -30,6 +32,8 @@ static void SetupBenchArgs(ArgsManager& argsman) argsman.AddArg("-output-csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-output-json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-priority-level=<l1,l2,l3>", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'", + benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } // parses a comma separated list like "10,20,30,50" @@ -45,6 +49,14 @@ static std::vector<double> parseAsymptote(const std::string& str) { return numbers; } +static uint8_t parsePriorityLevel(const std::string& str) { + uint8_t levels{0}; + for (const auto& level: SplitString(str, ',')) { + levels |= benchmark::StringToPriority(level); + } + return levels; +} + int main(int argc, char** argv) { ArgsManager argsman; @@ -106,16 +118,22 @@ int main(int argc, char** argv) return EXIT_SUCCESS; } - benchmark::Args args; - args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); - args.is_list_only = argsman.GetBoolArg("-list", false); - args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS)); - args.output_csv = argsman.GetPathArg("-output-csv"); - args.output_json = argsman.GetPathArg("-output-json"); - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); - args.sanity_check = argsman.GetBoolArg("-sanity-check", false); + try { + benchmark::Args args; + args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS)); + args.output_csv = argsman.GetPathArg("-output-csv"); + args.output_json = argsman.GetPathArg("-output-json"); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + args.sanity_check = argsman.GetBoolArg("-sanity-check", false); + args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY)); - benchmark::BenchRunner::RunAll(args); + benchmark::BenchRunner::RunAll(args); - return EXIT_SUCCESS; + return EXIT_SUCCESS; + } catch (const std::exception& e) { + tfm::format(std::cerr, "Error: %s\n", e.what()); + return EXIT_FAILURE; + } } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 4ed5397330..09be011fda 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -47,4 +47,4 @@ static void AssembleBlock(benchmark::Bench& bench) }); } -BENCHMARK(AssembleBlock); +BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 22dfb7aa5b..5d55ed9332 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -51,4 +51,4 @@ static void CCoinsCaching(benchmark::Bench& bench) ECC_Stop(); } -BENCHMARK(CCoinsCaching); +BENCHMARK(CCoinsCaching, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index a6f4eec4ca..9584dd58bb 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -39,6 +39,6 @@ static void CHACHA20_1MB(benchmark::Bench& bench) CHACHA20(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_64BYTES); -BENCHMARK(CHACHA20_256BYTES); -BENCHMARK(CHACHA20_1MB); +BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index e994279a4d..15b3a4f310 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -115,12 +115,12 @@ static void HASH_1MB(benchmark::Bench& bench) HASH(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT); -BENCHMARK(HASH_64BYTES); -BENCHMARK(HASH_256BYTES); -BENCHMARK(HASH_1MB); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 53aa470042..747279c161 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -50,5 +50,5 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) }); } -BENCHMARK(DeserializeBlockTest); -BENCHMARK(DeserializeAndCheckBlockTest); +BENCHMARK(DeserializeBlockTest, benchmark::PriorityLevel::HIGH); +BENCHMARK(DeserializeAndCheckBlockTest, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 602081fb9b..8517c9fee2 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -70,4 +70,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) queue.StopWorkerThreads(); ECC_Stop(); } -BENCHMARK(CCheckQueueSpeedPrevectorJob); +BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 6ada28115e..53d89039a7 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -121,5 +121,5 @@ static void BnBExhaustion(benchmark::Bench& bench) }); } -BENCHMARK(CoinSelection); -BENCHMARK(BnBExhaustion); +BENCHMARK(CoinSelection, benchmark::PriorityLevel::HIGH); +BENCHMARK(BnBExhaustion, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index d17ec503e7..162b02f344 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -150,19 +150,19 @@ static void MuHashPrecompute(benchmark::Bench& bench) }); } -BENCHMARK(RIPEMD160); -BENCHMARK(SHA1); -BENCHMARK(SHA256); -BENCHMARK(SHA512); -BENCHMARK(SHA3_256_1M); - -BENCHMARK(SHA256_32b); -BENCHMARK(SipHash_32b); -BENCHMARK(SHA256D64_1024); -BENCHMARK(FastRandom_32bit); -BENCHMARK(FastRandom_1bit); - -BENCHMARK(MuHash); -BENCHMARK(MuHashMul); -BENCHMARK(MuHashDiv); -BENCHMARK(MuHashPrecompute); +BENCHMARK(RIPEMD160, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA3_256_1M, benchmark::PriorityLevel::HIGH); + +BENCHMARK(SHA256_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SipHash_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256D64_1024, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_32bit, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_1bit, benchmark::PriorityLevel::HIGH); + +BENCHMARK(MuHash, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashMul, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashDiv, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashPrecompute, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index 5c868a8573..972a6ff953 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <key.h> +#include <pubkey.h> #include <script/descriptor.h> #include <script/standard.h> @@ -12,6 +13,9 @@ static void ExpandDescriptor(benchmark::Bench& bench) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; const std::pair<int64_t, int64_t> range = {0, 1000}; FlatSigningProvider provider; @@ -25,6 +29,8 @@ static void ExpandDescriptor(benchmark::Bench& bench) assert(success); } }); + + ECC_Stop(); } -BENCHMARK(ExpandDescriptor); +BENCHMARK(ExpandDescriptor, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 02a2e689b1..559854ff48 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -62,4 +62,4 @@ static void DuplicateInputs(benchmark::Bench& bench) }); } -BENCHMARK(DuplicateInputs); +BENCHMARK(DuplicateInputs, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index 72a9922e94..abef69cc42 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -18,4 +18,4 @@ static void Trig(benchmark::Bench& bench) }); } -BENCHMARK(Trig); +BENCHMARK(Trig, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 80babb213b..b795ebff39 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -81,8 +81,8 @@ static void GCSFilterMatch(benchmark::Bench& bench) filter.Match(GCSFilter::Element()); }); } -BENCHMARK(GCSBlockFilterGetHash); -BENCHMARK(GCSFilterConstruct); -BENCHMARK(GCSFilterDecode); -BENCHMARK(GCSFilterDecodeSkipCheck); -BENCHMARK(GCSFilterMatch); +BENCHMARK(GCSBlockFilterGetHash, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterConstruct, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterDecode, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterDecodeSkipCheck, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterMatch, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/hashpadding.cpp b/src/bench/hashpadding.cpp index 753c8c2881..ac5aeebe51 100644 --- a/src/bench/hashpadding.cpp +++ b/src/bench/hashpadding.cpp @@ -26,7 +26,7 @@ static void PrePadded(benchmark::Bench& bench) }); } -BENCHMARK(PrePadded); +BENCHMARK(PrePadded, benchmark::PriorityLevel::HIGH); static void RegularPadded(benchmark::Bench& bench) { @@ -44,4 +44,4 @@ static void RegularPadded(benchmark::Bench& bench) }); } -BENCHMARK(RegularPadded); +BENCHMARK(RegularPadded, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index b6d8824aba..ac8262654c 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -39,4 +39,4 @@ static void BenchLockedPool(benchmark::Bench& bench) addr.clear(); } -BENCHMARK(BenchLockedPool); +BENCHMARK(BenchLockedPool, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp index d28777df9e..49a9e59893 100644 --- a/src/bench/logging.cpp +++ b/src/bench/logging.cpp @@ -41,8 +41,8 @@ static void LoggingNoFile(benchmark::Bench& bench) }); } -BENCHMARK(LoggingYoThreadNames); -BENCHMARK(LoggingNoThreadNames); -BENCHMARK(LoggingYoCategory); -BENCHMARK(LoggingNoCategory); -BENCHMARK(LoggingNoFile); +BENCHMARK(LoggingYoThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingNoThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingYoCategory, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingNoCategory, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingNoFile, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 60d991fab9..878e375a7c 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -132,4 +132,4 @@ static void MempoolEviction(benchmark::Bench& bench) }); } -BENCHMARK(MempoolEviction); +BENCHMARK(MempoolEviction, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 725a6f8f5b..9f5b28dca7 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -114,5 +114,5 @@ static void MempoolCheck(benchmark::Bench& bench) }); } -BENCHMARK(ComplexMemPool); -BENCHMARK(MempoolCheck); +BENCHMARK(ComplexMemPool, benchmark::PriorityLevel::HIGH); +BENCHMARK(MempoolCheck, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index ba6629b9f0..4140d67bc7 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -23,4 +23,4 @@ static void MerkleRoot(benchmark::Bench& bench) }); } -BENCHMARK(MerkleRoot); +BENCHMARK(MerkleRoot, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp index f05f5e8f64..c3e3670de7 100644 --- a/src/bench/peer_eviction.cpp +++ b/src/bench/peer_eviction.cpp @@ -141,15 +141,15 @@ static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) // - 250 candidates is the number of peers reported by operators of busy nodes // No disadvantaged networks, with 250 eviction candidates. -BENCHMARK(EvictionProtection0Networks250Candidates); +BENCHMARK(EvictionProtection0Networks250Candidates, benchmark::PriorityLevel::HIGH); // 1 disadvantaged network (Tor) with 250 eviction candidates. -BENCHMARK(EvictionProtection1Networks250Candidates); +BENCHMARK(EvictionProtection1Networks250Candidates, benchmark::PriorityLevel::HIGH); // 2 disadvantaged networks (I2P, Tor) with 250 eviction candidates. -BENCHMARK(EvictionProtection2Networks250Candidates); +BENCHMARK(EvictionProtection2Networks250Candidates, benchmark::PriorityLevel::HIGH); // 3 disadvantaged networks (I2P/localhost/Tor) with 50/100/250 eviction candidates. -BENCHMARK(EvictionProtection3Networks050Candidates); -BENCHMARK(EvictionProtection3Networks100Candidates); -BENCHMARK(EvictionProtection3Networks250Candidates); +BENCHMARK(EvictionProtection3Networks050Candidates, benchmark::PriorityLevel::HIGH); +BENCHMARK(EvictionProtection3Networks100Candidates, benchmark::PriorityLevel::HIGH); +BENCHMARK(EvictionProtection3Networks250Candidates, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp index cdef97c0ea..ad5a72ffde 100644 --- a/src/bench/poly1305.cpp +++ b/src/bench/poly1305.cpp @@ -36,6 +36,6 @@ static void POLY1305_1MB(benchmark::Bench& bench) POLY1305(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(POLY1305_64BYTES); -BENCHMARK(POLY1305_256BYTES); -BENCHMARK(POLY1305_1MB); +BENCHMARK(POLY1305_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(POLY1305_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(POLY1305_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index b3688bab1b..9e2e7d11c4 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -85,12 +85,12 @@ static void PrevectorDeserialize(benchmark::Bench& bench) { \ Prevector##name<nontrivial_t>(bench); \ } \ - BENCHMARK(Prevector##name##Nontrivial); \ + BENCHMARK(Prevector##name##Nontrivial, benchmark::PriorityLevel::HIGH); \ static void Prevector##name##Trivial(benchmark::Bench& bench) \ { \ Prevector##name<trivial_t>(bench); \ } \ - BENCHMARK(Prevector##name##Trivial); + BENCHMARK(Prevector##name##Trivial, benchmark::PriorityLevel::HIGH); PREVECTOR_TEST(Clear) PREVECTOR_TEST(Destructor) diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 8f05e3bad0..865d99f9e8 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -32,5 +32,5 @@ static void RollingBloomReset(benchmark::Bench& bench) }); } -BENCHMARK(RollingBloom); -BENCHMARK(RollingBloomReset); +BENCHMARK(RollingBloom, benchmark::PriorityLevel::HIGH); +BENCHMARK(RollingBloomReset, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index e6fc8d21f4..5a178f308a 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -45,7 +45,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) }); } -BENCHMARK(BlockToJsonVerbose); +BENCHMARK(BlockToJsonVerbose, benchmark::PriorityLevel::HIGH); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { @@ -57,4 +57,4 @@ static void BlockToJsonVerboseWrite(benchmark::Bench& bench) }); } -BENCHMARK(BlockToJsonVerboseWrite); +BENCHMARK(BlockToJsonVerboseWrite, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 0e6fdae3d7..4fdc31ae05 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -40,4 +40,4 @@ static void RpcMempool(benchmark::Bench& bench) }); } -BENCHMARK(RpcMempool); +BENCHMARK(RpcMempool, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/strencodings.cpp b/src/bench/strencodings.cpp index 69b3a83cbf..16d14278a7 100644 --- a/src/bench/strencodings.cpp +++ b/src/bench/strencodings.cpp @@ -15,4 +15,4 @@ static void HexStrBench(benchmark::Bench& bench) }); } -BENCHMARK(HexStrBench); +BENCHMARK(HexStrBench, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index afc733482e..8256b5f7ba 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -36,7 +36,7 @@ static void BenchTimeMillisSys(benchmark::Bench& bench) }); } -BENCHMARK(BenchTimeDeprecated); -BENCHMARK(BenchTimeMillis); -BENCHMARK(BenchTimeMillisSys); -BENCHMARK(BenchTimeMock); +BENCHMARK(BenchTimeDeprecated, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchTimeMillis, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchTimeMillisSys, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchTimeMock, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 8e4708f260..f0e9db8ba1 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -96,5 +96,5 @@ static void VerifyNestedIfScript(benchmark::Bench& bench) }); } -BENCHMARK(VerifyScriptBench); -BENCHMARK(VerifyNestedIfScript); +BENCHMARK(VerifyScriptBench, benchmark::PriorityLevel::HIGH); +BENCHMARK(VerifyNestedIfScript, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index a5dd2f3bb1..22d99c0e29 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -57,7 +57,7 @@ static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, / static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/true); } static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/false); } -BENCHMARK(WalletBalanceDirty); -BENCHMARK(WalletBalanceClean); -BENCHMARK(WalletBalanceMine); -BENCHMARK(WalletBalanceWatch); +BENCHMARK(WalletBalanceDirty, benchmark::PriorityLevel::HIGH); +BENCHMARK(WalletBalanceClean, benchmark::PriorityLevel::HIGH); +BENCHMARK(WalletBalanceMine, benchmark::PriorityLevel::HIGH); +BENCHMARK(WalletBalanceWatch, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 27e4dd015d..8bfaf3044b 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -118,10 +118,10 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) #ifdef USE_BDB static void WalletLoadingLegacy(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/true); } -BENCHMARK(WalletLoadingLegacy); +BENCHMARK(WalletLoadingLegacy, benchmark::PriorityLevel::HIGH); #endif #ifdef USE_SQLITE static void WalletLoadingDescriptors(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/false); } -BENCHMARK(WalletLoadingDescriptors); +BENCHMARK(WalletLoadingDescriptors, benchmark::PriorityLevel::HIGH); #endif diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index e64e2202ba..6d77385584 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -642,7 +642,7 @@ public: " send Time since last message sent to the peer, in seconds\n" " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" - " \"*\" - the peer requested we not relay transactions to it (relaytxes is false)\n" + " \"*\" - whether we relay transactions to this peer (relaytxes is false)\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" " hb High-bandwidth BIP152 compact block relay\n" " \".\" (to) - we selected the peer as a high-bandwidth peer\n" diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp new file mode 100644 index 0000000000..6ad9f75b5d --- /dev/null +++ b/src/common/run_command.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <common/run_command.h> + +#include <tinyformat.h> +#include <univalue.h> + +#ifdef ENABLE_EXTERNAL_SIGNER +#if defined(__GNUC__) +// Boost 1.78 requires the following workaround. +// See: https://github.com/boostorg/process/issues/235 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif +#include <boost/process.hpp> +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif +#endif // ENABLE_EXTERNAL_SIGNER + +UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) +{ +#ifdef ENABLE_EXTERNAL_SIGNER + namespace bp = boost::process; + + UniValue result_json; + bp::opstream stdin_stream; + bp::ipstream stdout_stream; + bp::ipstream stderr_stream; + + if (str_command.empty()) return UniValue::VNULL; + + bp::child c( + str_command, + bp::std_out > stdout_stream, + bp::std_err > stderr_stream, + bp::std_in < stdin_stream + ); + if (!str_std_in.empty()) { + stdin_stream << str_std_in << std::endl; + } + stdin_stream.pipe().close(); + + std::string result; + std::string error; + std::getline(stdout_stream, result); + std::getline(stderr_stream, error); + + c.wait(); + const int n_error = c.exit_code(); + if (n_error) throw std::runtime_error(strprintf("RunCommandParseJSON error: process(%s) returned %d: %s\n", str_command, n_error, error)); + if (!result_json.read(result)) throw std::runtime_error("Unable to parse JSON: " + result); + + return result_json; +#else + throw std::runtime_error("Compiled without external signing support (required for external signing)."); +#endif // ENABLE_EXTERNAL_SIGNER +} diff --git a/src/common/run_command.h b/src/common/run_command.h new file mode 100644 index 0000000000..2fbdc071ee --- /dev/null +++ b/src/common/run_command.h @@ -0,0 +1,21 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_RUN_COMMAND_H +#define BITCOIN_COMMON_RUN_COMMAND_H + +#include <string> + +class UniValue; + +/** + * Execute a command which returns JSON, and parse the result. + * + * @param str_command The command to execute, including any arguments + * @param str_std_in string to pass to stdin + * @return parsed JSON + */ +UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in=""); + +#endif // BITCOIN_COMMON_RUN_COMMAND_H diff --git a/src/compat/compat.h b/src/compat/compat.h index a8e5552c0a..cc37797577 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -109,14 +109,6 @@ typedef char* sockopt_arg_type; #define USE_POLL #endif -bool static inline IsSelectableSocket(const SOCKET& s) { -#if defined(USE_POLL) || defined(WIN32) - return true; -#else - return (s < FD_SETSIZE); -#endif -} - // MSG_NOSIGNAL is not available on some platforms, if it doesn't exist define it as 0 #if !defined(MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 4dbc839941..7f45e35aef 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -128,7 +128,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) } CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) - : m_name{fs::PathToString(path.stem())} + : m_name{fs::PathToString(path.stem())}, m_path{path}, m_is_memory{fMemory} { penv = nullptr; readoptions.verify_checksums = true; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 665eaa0e98..1052da01d5 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -39,6 +39,10 @@ public: class CDBWrapper; +namespace dbwrapper { + using leveldb::DestroyDB; +} + /** These should be considered an implementation detail of the specific database. */ namespace dbwrapper_private { @@ -219,6 +223,12 @@ private: std::vector<unsigned char> CreateObfuscateKey() const; + //! path to filesystem storage + const fs::path m_path; + + //! whether or not the database resides in memory + bool m_is_memory; + public: /** * @param[in] path Location in the filesystem where leveldb data will be stored. @@ -268,6 +278,14 @@ public: return WriteBatch(batch, fSync); } + //! @returns filesystem path to the on-disk data. + std::optional<fs::path> StoragePath() { + if (m_is_memory) { + return {}; + } + return m_path; + } + template <typename K> bool Exists(const K& key) const { diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 094314e878..0e582629f7 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -3,10 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <common/run_command.h> #include <core_io.h> #include <psbt.h> #include <util/strencodings.h> -#include <util/system.h> #include <external_signer.h> #include <algorithm> diff --git a/src/index/base.cpp b/src/index/base.cpp index 88c2ce98fa..3eea09b17d 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -298,6 +298,10 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const } interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex, block.get()); if (CustomAppend(block_info)) { + // Setting the best block index is intentionally the last step of this + // function, so BlockUntilSyncedToCurrentChain callers waiting for the + // best block index to be updated can rely on the block being fully + // processed, and the index object being safe to delete. SetBestBlockIndex(pindex); } else { FatalError("%s: Failed to write block %s to index", @@ -414,10 +418,17 @@ IndexSummary BaseIndex::GetSummary() const void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) { assert(!node::fPruneMode || AllowPrune()); - m_best_block_index = block; if (AllowPrune() && block) { node::PruneLockInfo prune_lock; prune_lock.height_first = block->nHeight; WITH_LOCK(::cs_main, m_chainstate->m_blockman.UpdatePruneLock(GetName(), prune_lock)); } + + // Intentionally set m_best_block_index as the last step in this function, + // after updating prune locks above, and after making any other references + // to *this, so the BlockUntilSyncedToCurrentChain function (which checks + // m_best_block_index as an optimization) can be used to wait for the last + // BlockConnected notification and safely assume that prune locks are + // updated and that the index object is safe to delete. + m_best_block_index = block; } diff --git a/src/init.cpp b/src/init.cpp index 25b40c6c6e..cf4102e6a7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -45,6 +45,7 @@ #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> +#include <node/txreconciliation.h> #include <node/validation_cache_args.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -484,6 +485,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); @@ -1255,6 +1257,51 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // as they would never get updated. if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(args)); + // Check port numbers + for (const std::string port_option : { + "-port", + "-rpcport", + }) { + if (args.IsArgSet(port_option)) { + const std::string port = args.GetArg(port_option, ""); + uint16_t n; + if (!ParseUInt16(port, &n) || n == 0) { + return InitError(InvalidPortErrMsg(port_option, port)); + } + } + } + + for (const std::string port_option : { + "-i2psam", + "-onion", + "-proxy", + "-rpcbind", + "-torcontrol", + "-whitebind", + "-zmqpubhashblock", + "-zmqpubhashtx", + "-zmqpubrawblock", + "-zmqpubrawtx", + "-zmqpubsequence", + }) { + for (const std::string& socket_addr : args.GetArgs(port_option)) { + std::string host_out; + uint16_t port_out{0}; + if (!SplitHostPort(socket_addr, port_out, host_out)) { + return InitError(InvalidPortErrMsg(port_option, socket_addr)); + } + } + } + + for (const std::string& socket_addr : args.GetArgs("-bind")) { + std::string host_out; + uint16_t port_out{0}; + std::string bind_socket_addr = socket_addr.substr(0, socket_addr.rfind('=')); + if (!SplitHostPort(bind_socket_addr, port_out, host_out)) { + return InitError(InvalidPortErrMsg("-bind", socket_addr)); + } + } + // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : args.GetArgs("-uacomment")) { @@ -1663,7 +1710,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const auto BadPortWarning = [](const char* prefix, uint16_t port) { return strprintf(_("%s request to listen on port %u. This port is considered \"bad\" and " - "thus it is unlikely that any Bitcoin Core peers connect to it. See " + "thus it is unlikely that any peer will connect to it. See " "doc/p2p-bad-ports.md for details and a full list."), prefix, port); diff --git a/src/kernel/mempool_limits.h b/src/kernel/mempool_limits.h index e192e7e6cd..8d4495c3cb 100644 --- a/src/kernel/mempool_limits.h +++ b/src/kernel/mempool_limits.h @@ -24,6 +24,15 @@ struct MemPoolLimits { int64_t descendant_count{DEFAULT_DESCENDANT_LIMIT}; //! The maximum allowed size in virtual bytes of an entry and its descendants within a package. int64_t descendant_size_vbytes{DEFAULT_DESCENDANT_SIZE_LIMIT_KVB * 1'000}; + + /** + * @return MemPoolLimits with all the limits set to the maximum + */ + static constexpr MemPoolLimits NoLimits() + { + int64_t no_limit{std::numeric_limits<int64_t>::max()}; + return {no_limit, no_limit, no_limit, no_limit}; + } }; } // namespace kernel diff --git a/src/logging.cpp b/src/logging.cpp index a3f1d39be5..c95c0b7e37 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -180,6 +180,7 @@ const CLogCategoryDesc LogCategories[] = #endif {BCLog::UTIL, "util"}, {BCLog::BLOCKSTORE, "blockstorage"}, + {BCLog::TXRECONCILIATION, "txreconciliation"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -280,6 +281,8 @@ std::string LogCategoryToStr(BCLog::LogFlags category) return "util"; case BCLog::LogFlags::BLOCKSTORE: return "blockstorage"; + case BCLog::LogFlags::TXRECONCILIATION: + return "txreconciliation"; case BCLog::LogFlags::ALL: return "all"; } diff --git a/src/logging.h b/src/logging.h index fe91ee43a5..5ee6665c76 100644 --- a/src/logging.h +++ b/src/logging.h @@ -66,6 +66,7 @@ namespace BCLog { #endif UTIL = (1 << 25), BLOCKSTORE = (1 << 26), + TXRECONCILIATION = (1 << 27), ALL = ~(uint32_t)0, }; enum class Level { diff --git a/src/net.cpp b/src/net.cpp index 0736f3ef1b..3c28b9eddf 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -970,8 +970,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, return; } - if (!IsSelectableSocket(sock->Get())) - { + if (!sock->IsSelectable()) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); return; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a753d9b1a8..34b4840eb1 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -20,6 +20,7 @@ #include <netbase.h> #include <netmessagemaker.h> #include <node/blockstorage.h> +#include <node/txreconciliation.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -271,12 +272,7 @@ struct Peer { struct TxRelay { mutable RecursiveMutex m_bloom_filter_mutex; - /** Whether the peer wishes to receive transaction announcements. - * - * This is initially set based on the fRelay flag in the received - * `version` message. If initially set to false, it can only be flipped - * to true if we have offered the peer NODE_BLOOM services and it sends - * us a `filterload` or `filterclear` message. See BIP37. */ + /** Whether we relay transactions to this peer. */ bool m_relay_txs GUARDED_BY(m_bloom_filter_mutex){false}; /** A bloom filter for which transactions to announce to the peer. See BIP37. */ std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; @@ -708,6 +704,7 @@ private: ChainstateManager& m_chainman; CTxMemPool& m_mempool; TxRequestTracker m_txrequest GUARDED_BY(::cs_main); + std::unique_ptr<TxReconciliationTracker> m_txreconciliation; /** The height of the best chain */ std::atomic<int> m_best_height{-1}; @@ -1497,6 +1494,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) } WITH_LOCK(g_cs_orphans, m_orphanage.EraseForPeer(nodeid)); m_txrequest.DisconnectedPeer(nodeid); + if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; m_peers_downloading_from -= (state->nBlocksInFlight != 0); assert(m_peers_downloading_from >= 0); @@ -1781,6 +1779,11 @@ PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, m_mempool(pool), m_ignore_incoming_txs(ignore_incoming_txs) { + // While Erlay support is incomplete, it must be enabled explicitly via -txreconciliation. + // This argument can go away after Erlay support is complete. + if (gArgs.GetBoolArg("-txreconciliation", DEFAULT_TXRECONCILIATION_ENABLE)) { + m_txreconciliation = std::make_unique<TxReconciliationTracker>(TXRECONCILIATION_VERSION); + } } void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) @@ -3241,8 +3244,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); } - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); - pfrom.m_has_all_wanted_services = HasAllDesirableServiceFlags(nServices); peer->m_their_services = nServices; pfrom.SetAddrLocal(addrMe); @@ -3252,10 +3253,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } peer->m_starting_height = starting_height; - // We only initialize the m_tx_relay data structure if: - // - this isn't an outbound block-relay-only connection; and - // - fRelay=true or we're offering NODE_BLOOM to this peer - // (NODE_BLOOM means that the peer may turn on tx relay later) + // We only initialize the Peer::TxRelay m_relay_txs data structure if: + // - this isn't an outbound block-relay-only connection, and + // - fRelay=true (the peer wishes to receive transaction announcements) + // or we're offering NODE_BLOOM to this peer. NODE_BLOOM means that + // the peer may turn on transaction relay later. if (!pfrom.IsBlockOnlyConn() && (fRelay || (peer->m_our_services & NODE_BLOOM))) { auto* const tx_relay = peer->SetTxRelay(); @@ -3266,6 +3268,25 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fRelay) pfrom.m_relays_txs = true; } + if (greatest_common_version >= WTXID_RELAY_VERSION && m_txreconciliation) { + // Per BIP-330, we announce txreconciliation support if: + // - protocol version per the VERSION message supports WTXID_RELAY; + // - we intended to exchange transactions over this connection while establishing it + // and the peer indicated support for transaction relay in the VERSION message; + // - we are not in -blocksonly mode. + if (pfrom.m_relays_txs && !m_ignore_incoming_txs) { + const uint64_t recon_salt = m_txreconciliation->PreRegisterPeer(pfrom.GetId()); + // We suggest our txreconciliation role (initiator/responder) based on + // the connection direction. + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDTXRCNCL, + !pfrom.IsInboundConn(), + pfrom.IsInboundConn(), + TXRECONCILIATION_VERSION, recon_salt)); + } + } + + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); + // Potentially mark this peer as a preferred download peer. { LOCK(cs_main); @@ -3393,6 +3414,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // they may wish to request compact blocks from us m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION)); } + + if (m_txreconciliation) { + if (!peer->m_wtxid_relay || !m_txreconciliation->IsPeerRegistered(pfrom.GetId())) { + // We could have optimistically pre-registered/registered the peer. In that case, + // we should forget about the reconciliation state here if this wasn't followed + // by WTXIDRELAY (since WTXIDRELAY can't be announced later). + m_txreconciliation->ForgetPeer(pfrom.GetId()); + } + } + pfrom.fSuccessfullyConnected = true; return; } @@ -3456,6 +3487,58 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Received from a peer demonstrating readiness to announce transactions via reconciliations. + // This feature negotiation must happen between VERSION and VERACK to avoid relay problems + // from switching announcement protocols after the connection is up. + if (msg_type == NetMsgType::SENDTXRCNCL) { + if (!m_txreconciliation) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl from peer=%d ignored, as our node does not have txreconciliation enabled\n", pfrom.GetId()); + return; + } + + if (pfrom.fSuccessfullyConnected) { + // Disconnect peers that send a SENDTXRCNCL message after VERACK. + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received after verack from peer=%d; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + if (!peer->GetTxRelay()) { + // Disconnect peers that send a SENDTXRCNCL message even though we indicated we don't + // support transaction relay. + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received from peer=%d to which we indicated no tx relay; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + bool is_peer_initiator, is_peer_responder; + uint32_t peer_txreconcl_version; + uint64_t remote_salt; + vRecv >> is_peer_initiator >> is_peer_responder >> peer_txreconcl_version >> remote_salt; + + if (m_txreconciliation->IsPeerRegistered(pfrom.GetId())) { + // A peer is already registered, meaning we already received SENDTXRCNCL from them. + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d (sendtxrcncl received from already registered peer); disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + const ReconciliationRegisterResult result = m_txreconciliation->RegisterPeer(pfrom.GetId(), pfrom.IsInboundConn(), + is_peer_initiator, is_peer_responder, + peer_txreconcl_version, + remote_salt); + + // If it's a protocol violation, disconnect. + // If the peer was not found (but something unexpected happened) or it was registered, + // nothing to be done. + if (result == ReconciliationRegisterResult::PROTOCOL_VIOLATION) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + return; + } + if (!pfrom.fSuccessfullyConnected) { LogPrint(BCLog::NET, "Unsupported message \"%s\" prior to verack from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return; diff --git a/src/netbase.cpp b/src/netbase.cpp index a5f7bda875..8169b40ea6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -304,8 +304,7 @@ enum class IntrRecvError { * read. * * @see This function can be interrupted by calling InterruptSocks5(bool). - * Sockets can be made non-blocking with SetSocketNonBlocking(const - * SOCKET&). + * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { @@ -503,7 +502,7 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) // Ensure that waiting for I/O on this socket won't result in undefined // behavior. - if (!IsSelectableSocket(sock->Get())) { + if (!sock->IsSelectable()) { LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); return nullptr; } @@ -525,7 +524,7 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) } // Set the non-blocking option on the socket. - if (!SetSocketNonBlocking(sock->Get())) { + if (!sock->SetNonBlocking()) { LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); return nullptr; } @@ -717,21 +716,6 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) return false; } -bool SetSocketNonBlocking(const SOCKET& hSocket) -{ -#ifdef WIN32 - u_long nOne = 1; - if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { -#else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { -#endif - return false; - } - - return true; -} - void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index fadc8b418e..f7816f5d1d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -221,8 +221,6 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -/** Enable non-blocking mode for a socket */ -bool SetSocketNonBlocking(const SOCKET& hSocket); void InterruptSocks5(bool interrupt); /** diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 57f81e6bb6..04d46f4361 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -524,6 +524,16 @@ void BlockManager::FlushUndoFile(int block_file, bool finalize) void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) { LOCK(cs_LastBlockFile); + + if (m_blockfile_info.size() < 1) { + // Return if we haven't loaded any blockfiles yet. This happens during + // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which + // then calls FlushStateToDisk()), resulting in a call to this function before we + // have populated `m_blockfile_info` via LoadBlockIndexDB(). + return; + } + assert(static_cast<int>(m_blockfile_info.size()) > m_last_blockfile); + FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); @@ -789,19 +799,24 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c return true; } -/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; - if (dbp != nullptr) { + const auto position_known {dbp != nullptr}; + if (position_known) { blockPos = *dbp; + } else { + // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for + // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). + // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. + nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } - if (dbp == nullptr) { + if (!position_known) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 37d74ed102..29501c1959 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -44,6 +44,9 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB +/** Size of header written by WriteBlockToDisk before a serialized CBlock */ +static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); + extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; /** Pruning-related variables and constants */ @@ -171,6 +174,7 @@ public: bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); /** Calculate the amount of disk space the block & undo files currently use */ diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 3f1d6dd743..26af523491 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -48,10 +48,15 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } LOCK(cs_main); - chainman.InitializeChainstate(options.mempool); chainman.m_total_coinstip_cache = cache_sizes.coins; chainman.m_total_coinsdb_cache = cache_sizes.coins_db; + // Load the fully validated chainstate. + chainman.InitializeChainstate(options.mempool); + + // Load a chain created from a UTXO snapshot, if any exist. + chainman.DetectSnapshotChainstate(options.mempool); + auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -98,12 +103,20 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } + // Conservative value which is arbitrarily chosen, as it will ultimately be changed + // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure + // that the sum of the two caches (40%) does not exceed the allowable amount + // during this temporary initialization state. + double init_cache_fraction = 0.2; + // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! for (Chainstate* chainstate : chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( - /*cache_size_bytes=*/cache_sizes.coins_db, + /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, /*should_wipe=*/options.reindex || options.reindex_chainstate); @@ -125,7 +138,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(cache_sizes.coins); + chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { @@ -146,6 +159,11 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize }; } + // Now that chainstates are loaded and we're able to flush to + // disk, rebalance the coins caches to desired levels based + // on the condition of each chainstate. + chainman.MaybeRebalanceCaches(); + return {ChainstateLoadStatus::SUCCESS, {}}; } diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index aa7ddec770..8a0011a629 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -660,8 +660,7 @@ public: std::string unused_error_string; LOCK(m_node.mempool->cs); return m_node.mempool->CalculateMemPoolAncestors( - entry, ancestors, limits.ancestor_count, limits.ancestor_size_vbytes, - limits.descendant_count, limits.descendant_size_vbytes, unused_error_string); + entry, ancestors, limits, unused_error_string); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override { diff --git a/src/node/miner.cpp b/src/node/miner.cpp index b277188c1f..e11ec5b0f1 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -105,7 +105,7 @@ void BlockAssembler::resetBlock() std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) { - int64_t nTimeStart = GetTimeMicros(); + const auto time_start{SteadyClock::now()}; resetBlock(); @@ -143,7 +143,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc addPackageTxs(*m_mempool, nPackagesSelected, nDescendantsUpdated); } - int64_t nTime1 = GetTimeMicros(); + const auto time_1{SteadyClock::now()}; m_last_block_num_txs = nBlockTx; m_last_block_weight = nBlockWeight; @@ -173,9 +173,12 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc if (!TestBlockValidity(state, chainparams, m_chainstate, *pblock, pindexPrev, GetAdjustedTime, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, state.ToString())); } - int64_t nTime2 = GetTimeMicros(); + const auto time_2{SteadyClock::now()}; - LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", 0.001 * (nTime1 - nTimeStart), nPackagesSelected, nDescendantsUpdated, 0.001 * (nTime2 - nTime1), 0.001 * (nTime2 - nTimeStart)); + LogPrint(BCLog::BENCH, "CreateNewBlock() packages: %.2fms (%d packages, %d updated descendants), validity: %.2fms (total %.2fms)\n", + Ticks<MillisecondsDouble>(time_1 - time_start), nPackagesSelected, nDescendantsUpdated, + Ticks<MillisecondsDouble>(time_2 - time_1), + Ticks<MillisecondsDouble>(time_2 - time_start)); return std::move(pblocktemplate); } @@ -257,13 +260,9 @@ static int UpdatePackagesForAdded(const CTxMemPool& mempool, modtxiter mit = mapModifiedTx.find(desc); if (mit == mapModifiedTx.end()) { CTxMemPoolModifiedEntry modEntry(desc); - modEntry.nSizeWithAncestors -= it->GetTxSize(); - modEntry.nModFeesWithAncestors -= it->GetModifiedFee(); - modEntry.nSigOpCostWithAncestors -= it->GetSigOpCost(); - mapModifiedTx.insert(modEntry); - } else { - mapModifiedTx.modify(mit, update_for_parent_inclusion(it)); + mit = mapModifiedTx.insert(modEntry).first; } + mapModifiedTx.modify(mit, update_for_parent_inclusion(it)); } } return nDescendantsUpdated; @@ -396,9 +395,8 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele } CTxMemPool::setEntries ancestors; - uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - mempool.CalculateMemPoolAncestors(*iter, ancestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); + mempool.CalculateMemPoolAncestors(*iter, ancestors, CTxMemPool::Limits::NoLimits(), dummy, false); onlyUnconfirmed(ancestors); ancestors.insert(iter); diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 57162cd679..ca3fc0955d 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -59,7 +59,7 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) } // Check if it is final - if (!utxo.IsNull() && !PSBTInputSigned(input)) { + if (!PSBTInputSignedAndVerified(psbtx, i, &txdata)) { input_analysis.is_final = false; // Figure out what is missing diff --git a/src/node/txreconciliation.cpp b/src/node/txreconciliation.cpp new file mode 100644 index 0000000000..974358fcda --- /dev/null +++ b/src/node/txreconciliation.cpp @@ -0,0 +1,184 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txreconciliation.h> + +#include <util/check.h> +#include <util/system.h> + +#include <unordered_map> +#include <variant> + + +namespace { + +/** Static salt component used to compute short txids for sketch construction, see BIP-330. */ +const std::string RECON_STATIC_SALT = "Tx Relay Salting"; +const HashWriter RECON_SALT_HASHER = TaggedHash(RECON_STATIC_SALT); + +/** + * Salt (specified by BIP-330) constructed from contributions from both peers. It is used + * to compute transaction short IDs, which are then used to construct a sketch representing a set + * of transactions we want to announce to the peer. + */ +uint256 ComputeSalt(uint64_t salt1, uint64_t salt2) +{ + // According to BIP-330, salts should be combined in ascending order. + return (HashWriter(RECON_SALT_HASHER) << std::min(salt1, salt2) << std::max(salt1, salt2)).GetSHA256(); +} + +/** + * Keeps track of txreconciliation-related per-peer state. + */ +class TxReconciliationState +{ +public: + /** + * TODO: This field is public to ignore -Wunused-private-field. Make private once used in + * the following commits. + * + * Reconciliation protocol assumes using one role consistently: either a reconciliation + * initiator (requesting sketches), or responder (sending sketches). This defines our role. + * + */ + bool m_we_initiate; + + /** + * TODO: These fields are public to ignore -Wunused-private-field. Make private once used in + * the following commits. + * + * These values are used to salt short IDs, which is necessary for transaction reconciliations. + */ + uint64_t m_k0, m_k1; + + TxReconciliationState(bool we_initiate, uint64_t k0, uint64_t k1) : m_we_initiate(we_initiate), m_k0(k0), m_k1(k1) {} +}; + +} // namespace + +/** Actual implementation for TxReconciliationTracker's data structure. */ +class TxReconciliationTracker::Impl +{ +private: + mutable Mutex m_txreconciliation_mutex; + + // Local protocol version + uint32_t m_recon_version; + + /** + * Keeps track of txreconciliation states of eligible peers. + * For pre-registered peers, the locally generated salt is stored. + * For registered peers, the locally generated salt is forgotten, and the state (including + * "full" salt) is stored instead. + */ + std::unordered_map<NodeId, std::variant<uint64_t, TxReconciliationState>> m_states GUARDED_BY(m_txreconciliation_mutex); + +public: + explicit Impl(uint32_t recon_version) : m_recon_version(recon_version) {} + + uint64_t PreRegisterPeer(NodeId peer_id) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + // We do not support txreconciliation salt/version updates. + assert(m_states.find(peer_id) == m_states.end()); + + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Pre-register peer=%d\n", peer_id); + const uint64_t local_salt{GetRand(UINT64_MAX)}; + + // We do this exactly once per peer (which are unique by NodeId, see GetNewNodeId) so it's + // safe to assume we don't have this record yet. + Assert(m_states.emplace(peer_id, local_salt).second); + return local_salt; + } + + ReconciliationRegisterResult RegisterPeer(NodeId peer_id, bool is_peer_inbound, bool is_peer_recon_initiator, + bool is_peer_recon_responder, uint32_t peer_recon_version, + uint64_t remote_salt) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + auto recon_state = m_states.find(peer_id); + + // A peer should be in the pre-registered state to proceed here. + if (recon_state == m_states.end()) return NOT_FOUND; + uint64_t* local_salt = std::get_if<uint64_t>(&recon_state->second); + // A peer is already registered. This should be checked by the caller. + Assume(local_salt); + + // If the peer supports the version which is lower than ours, we downgrade to the version + // it supports. For now, this only guarantees that nodes with future reconciliation + // versions have the choice of reconciling with this current version. However, they also + // have the choice to refuse supporting reconciliations if the common version is not + // satisfactory (e.g. too low). + const uint32_t recon_version{std::min(peer_recon_version, m_recon_version)}; + // v1 is the lowest version, so suggesting something below must be a protocol violation. + if (recon_version < 1) return PROTOCOL_VIOLATION; + + // Must match SENDTXRCNCL logic. + const bool they_initiate = is_peer_recon_initiator && is_peer_inbound; + const bool we_initiate = !is_peer_inbound && is_peer_recon_responder; + + // If we ever announce support for both requesting and responding, this will need + // tie-breaking. For now, this is mutually exclusive because both are based on the + // inbound flag. + assert(!(they_initiate && we_initiate)); + + // The peer set both flags to false, we treat it as a protocol violation. + if (!(they_initiate || we_initiate)) return PROTOCOL_VIOLATION; + + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Register peer=%d with the following params: " /* Continued */ + "we_initiate=%i, they_initiate=%i.\n", + peer_id, we_initiate, they_initiate); + + const uint256 full_salt{ComputeSalt(*local_salt, remote_salt)}; + recon_state->second = TxReconciliationState(we_initiate, full_salt.GetUint64(0), full_salt.GetUint64(1)); + return SUCCESS; + } + + void ForgetPeer(NodeId peer_id) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + if (m_states.erase(peer_id)) { + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Forget txreconciliation state of peer=%d\n", peer_id); + } + } + + bool IsPeerRegistered(NodeId peer_id) const EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + auto recon_state = m_states.find(peer_id); + return (recon_state != m_states.end() && + std::holds_alternative<TxReconciliationState>(recon_state->second)); + } +}; + +TxReconciliationTracker::TxReconciliationTracker(uint32_t recon_version) : m_impl{std::make_unique<TxReconciliationTracker::Impl>(recon_version)} {} + +TxReconciliationTracker::~TxReconciliationTracker() = default; + +uint64_t TxReconciliationTracker::PreRegisterPeer(NodeId peer_id) +{ + return m_impl->PreRegisterPeer(peer_id); +} + +ReconciliationRegisterResult TxReconciliationTracker::RegisterPeer(NodeId peer_id, bool is_peer_inbound, + bool is_peer_recon_initiator, bool is_peer_recon_responder, + uint32_t peer_recon_version, uint64_t remote_salt) +{ + return m_impl->RegisterPeer(peer_id, is_peer_inbound, is_peer_recon_initiator, is_peer_recon_responder, + peer_recon_version, remote_salt); +} + +void TxReconciliationTracker::ForgetPeer(NodeId peer_id) +{ + m_impl->ForgetPeer(peer_id); +} + +bool TxReconciliationTracker::IsPeerRegistered(NodeId peer_id) const +{ + return m_impl->IsPeerRegistered(peer_id); +} diff --git a/src/node/txreconciliation.h b/src/node/txreconciliation.h new file mode 100644 index 0000000000..a4f0870914 --- /dev/null +++ b/src/node/txreconciliation.h @@ -0,0 +1,90 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TXRECONCILIATION_H +#define BITCOIN_NODE_TXRECONCILIATION_H + +#include <net.h> +#include <sync.h> + +#include <memory> +#include <tuple> + +/** Whether transaction reconciliation protocol should be enabled by default. */ +static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; +/** Supported transaction reconciliation protocol version */ +static constexpr uint32_t TXRECONCILIATION_VERSION{1}; + +enum ReconciliationRegisterResult { + NOT_FOUND = 0, + SUCCESS = 1, + PROTOCOL_VIOLATION = 2, +}; + +/** + * Transaction reconciliation is a way for nodes to efficiently announce transactions. + * This object keeps track of all txreconciliation-related communications with the peers. + * The high-level protocol is: + * 0. Txreconciliation protocol handshake. + * 1. Once we receive a new transaction, add it to the set instead of announcing immediately. + * 2. At regular intervals, a txreconciliation initiator requests a sketch from a peer, where a + * sketch is a compressed representation of short form IDs of the transactions in their set. + * 3. Once the initiator received a sketch from the peer, the initiator computes a local sketch, + * and combines the two sketches to attempt finding the difference in *sets*. + * 4a. If the difference was not larger than estimated, see SUCCESS below. + * 4b. If the difference was larger than estimated, initial txreconciliation fails. The initiator + * requests a larger sketch via an extension round (allowed only once). + * - If extension succeeds (a larger sketch is sufficient), see SUCCESS below. + * - If extension fails (a larger sketch is insufficient), see FAILURE below. + * + * SUCCESS. The initiator knows full symmetrical difference and can request what the initiator is + * missing and announce to the peer what the peer is missing. + * + * FAILURE. The initiator notifies the peer about the failure and announces all transactions from + * the corresponding set. Once the peer received the failure notification, the peer + * announces all transactions from their set. + + * This is a modification of the Erlay protocol (https://arxiv.org/abs/1905.10518) with two + * changes (sketch extensions instead of bisections, and an extra INV exchange round), both + * are motivated in BIP-330. + */ +class TxReconciliationTracker +{ +private: + class Impl; + const std::unique_ptr<Impl> m_impl; + +public: + explicit TxReconciliationTracker(uint32_t recon_version); + ~TxReconciliationTracker(); + + /** + * Step 0. Generates initial part of the state (salt) required to reconcile txs with the peer. + * The salt is used for short ID computation required for txreconciliation. + * The function returns the salt. + * A peer can't participate in future txreconciliations without this call. + * This function must be called only once per peer. + */ + uint64_t PreRegisterPeer(NodeId peer_id); + + /** + * Step 0. Once the peer agreed to reconcile txs with us, generate the state required to track + * ongoing reconciliations. Must be called only after pre-registering the peer and only once. + */ + ReconciliationRegisterResult RegisterPeer(NodeId peer_id, bool is_peer_inbound, bool is_peer_recon_initiator, + bool is_peer_recon_responder, uint32_t peer_recon_version, uint64_t remote_salt); + + /** + * Attempts to forget txreconciliation-related state of the peer (if we previously stored any). + * After this, we won't be able to reconcile transactions with the peer. + */ + void ForgetPeer(NodeId peer_id); + + /** + * Check if a peer is registered to reconcile transactions with us. + */ + bool IsPeerRegistered(NodeId peer_id) const; +}; + +#endif // BITCOIN_NODE_TXRECONCILIATION_H diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp new file mode 100644 index 0000000000..bab1b75211 --- /dev/null +++ b/src/node/utxo_snapshot.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/utxo_snapshot.h> + +#include <fs.h> +#include <logging.h> +#include <streams.h> +#include <uint256.h> +#include <util/system.h> +#include <validation.h> + +#include <cstdio> +#include <optional> + +namespace node { + +bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) +{ + AssertLockHeld(::cs_main); + assert(snapshot_chainstate.m_from_snapshot_blockhash); + + const std::optional<fs::path> chaindir = snapshot_chainstate.CoinsDB().StoragePath(); + assert(chaindir); // Sanity check that chainstate isn't in-memory. + const fs::path write_to = *chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; + + FILE* file{fsbridge::fopen(write_to, "wb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + LogPrintf("[snapshot] failed to open base blockhash file for writing: %s\n", + fs::PathToString(write_to)); + return false; + } + afile << *snapshot_chainstate.m_from_snapshot_blockhash; + + if (afile.fclose() != 0) { + LogPrintf("[snapshot] failed to close base blockhash file %s after writing\n", + fs::PathToString(write_to)); + return false; + } + return true; +} + +std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) +{ + if (!fs::exists(chaindir)) { + LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " /* Continued */ + "exists at path %s\n", fs::PathToString(chaindir)); + return std::nullopt; + } + const fs::path read_from = chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; + const std::string read_from_str = fs::PathToString(read_from); + + if (!fs::exists(read_from)) { + LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " /* Continued */ + "exists at path %s. Try deleting %s and calling loadtxoutset again?\n", + fs::PathToString(chaindir), read_from_str); + return std::nullopt; + } + + uint256 base_blockhash; + FILE* file{fsbridge::fopen(read_from, "rb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + LogPrintf("[snapshot] failed to open base blockhash file for reading: %s\n", + read_from_str); + return std::nullopt; + } + afile >> base_blockhash; + + if (std::fgetc(afile.Get()) != EOF) { + LogPrintf("[snapshot] warning: unexpected trailing data in %s\n", read_from_str); + } else if (std::ferror(afile.Get())) { + LogPrintf("[snapshot] warning: i/o error reading %s\n", read_from_str); + } + return base_blockhash; +} + +std::optional<fs::path> FindSnapshotChainstateDir() +{ + fs::path possible_dir = + gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); + + if (fs::exists(possible_dir)) { + return possible_dir; + } + return std::nullopt; +} + +} // namespace node diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 9dd6f06997..c94521792f 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -6,8 +6,14 @@ #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include <fs.h> #include <uint256.h> #include <serialize.h> +#include <validation.h> + +#include <optional> + +extern RecursiveMutex cs_main; namespace node { //! Metadata describing a serialized version of a UTXO set from which an @@ -33,6 +39,33 @@ public: SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } }; + +//! The file in the snapshot chainstate dir which stores the base blockhash. This is +//! needed to reconstruct snapshot chainstates on init. +//! +//! Because we only allow loading a single snapshot at a time, there will only be one +//! chainstate directory with this filename present within it. +const fs::path SNAPSHOT_BLOCKHASH_FILENAME{"base_blockhash"}; + +//! Write out the blockhash of the snapshot base block that was used to construct +//! this chainstate. This value is read in during subsequent initializations and +//! used to reconstruct snapshot-based chainstates. +bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +//! Read the blockhash of the snapshot base block that was used to construct the +//! chainstate. +std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +//! Suffix appended to the chainstate (leveldb) dir when created based upon +//! a snapshot. +constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; + + +//! Return a path to the snapshot-based chainstate dir, if one exists. +std::optional<fs::path> FindSnapshotChainstateDir(); + } // namespace node #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 2b940be07e..ab5599a1b4 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -997,8 +997,9 @@ bool CBlockPolicyEstimator::Read(AutoFile& filein) return true; } -void CBlockPolicyEstimator::FlushUnconfirmed() { - int64_t startclear = GetTimeMicros(); +void CBlockPolicyEstimator::FlushUnconfirmed() +{ + const auto startclear{SteadyClock::now()}; LOCK(m_cs_fee_estimator); size_t num_entries = mapMemPoolTxs.size(); // Remove every entry in mapMemPoolTxs @@ -1006,24 +1007,41 @@ void CBlockPolicyEstimator::FlushUnconfirmed() { auto mi = mapMemPoolTxs.begin(); _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs } - int64_t endclear = GetTimeMicros(); - LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, (endclear - startclear)*0.000001); + const auto endclear{SteadyClock::now()}; + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear)); } -FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) +static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee, + double max_filter_fee_rate, + double fee_filter_spacing) { - CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); - feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { - feeset.insert(bucketBoundary); + std::set<double> fee_set; + + const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)}; + fee_set.insert(0); + for (double bucket_boundary = min_fee_limit; + bucket_boundary <= max_filter_fee_rate; + bucket_boundary *= fee_filter_spacing) { + + fee_set.insert(bucket_boundary); } + + return fee_set; +} + +FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) + : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} +{ } CAmount FeeFilterRounder::round(CAmount currentMinFee) { - std::set<double>::iterator it = feeset.lower_bound(currentMinFee); - if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { - it--; + AssertLockNotHeld(m_insecure_rand_mutex); + std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee); + if (it == m_fee_set.end() || + (it != m_fee_set.begin() && + WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { + --it; } return static_cast<CAmount>(*it); } diff --git a/src/policy/fees.h b/src/policy/fees.h index e4628bf853..204c4f2118 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -299,14 +299,15 @@ private: public: /** Create new FeeFilterRounder */ - explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); + explicit FeeFilterRounder(const CFeeRate& min_incremental_fee); - /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ - CAmount round(CAmount currentMinFee); + /** Quantize a minimum fee for privacy purpose before broadcast. */ + CAmount round(CAmount currentMinFee) EXCLUSIVE_LOCKS_REQUIRED(!m_insecure_rand_mutex); private: - std::set<double> feeset; - FastRandomContext insecure_rand; + const std::set<double> m_fee_set; + Mutex m_insecure_rand_mutex; + FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; #endif // BITCOIN_POLICY_FEES_H diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 6098caced9..55f47f485b 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -36,10 +36,9 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If all the inputs have nSequence >= maxint-1, it still might be // signaled for RBF if any unconfirmed parents have signaled. - uint64_t noLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; CTxMemPoolEntry entry = *pool.mapTx.find(tx.GetHash()); - pool.CalculateMemPoolAncestors(entry, ancestors, noLimit, noLimit, noLimit, noLimit, dummy, false); + pool.CalculateMemPoolAncestors(entry, ancestors, CTxMemPool::Limits::NoLimits(), dummy, false); for (CTxMemPool::txiter it : ancestors) { if (SignalsOptInRBF(it->GetTx())) { diff --git a/src/protocol.cpp b/src/protocol.cpp index bdd1cc2aff..23c68b335b 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -44,6 +44,7 @@ const char *CFHEADERS="cfheaders"; const char *GETCFCHECKPT="getcfcheckpt"; const char *CFCHECKPT="cfcheckpt"; const char *WTXIDRELAY="wtxidrelay"; +const char *SENDTXRCNCL="sendtxrcncl"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -84,6 +85,7 @@ const static std::string allNetMessageTypes[] = { NetMsgType::GETCFCHECKPT, NetMsgType::CFCHECKPT, NetMsgType::WTXIDRELAY, + NetMsgType::SENDTXRCNCL, }; const static std::vector<std::string> allNetMessageTypesVec(std::begin(allNetMessageTypes), std::end(allNetMessageTypes)); diff --git a/src/protocol.h b/src/protocol.h index b85dc0d820..17a363b1d3 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -258,6 +258,14 @@ extern const char* CFCHECKPT; * @since protocol version 70016 as described by BIP 339. */ extern const char* WTXIDRELAY; +/** + * Contains 2 1-byte bools, a 4-byte version number and an 8-byte salt. + * The 2 booleans indicate that a node is willing to participate in transaction + * reconciliation, respectively as an initiator or as a receiver. + * The salt is used to compute short txids needed for efficient + * txreconciliation, as described by BIP 330. + */ +extern const char* SENDTXRCNCL; }; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ diff --git a/src/psbt.cpp b/src/psbt.cpp index 36fec74bc9..461987c503 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -4,6 +4,7 @@ #include <psbt.h> +#include <policy/policy.h> #include <util/check.h> #include <util/strencodings.h> @@ -218,8 +219,14 @@ void PSBTOutput::FillSignatureData(SignatureData& sigdata) const for (const auto& key_pair : hd_keypaths) { sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); } - if (m_tap_tree.has_value() && m_tap_internal_key.IsFullyValid()) { - TaprootSpendData spenddata = m_tap_tree->GetSpendData(); + if (!m_tap_tree.empty() && m_tap_internal_key.IsFullyValid()) { + TaprootBuilder builder; + for (const auto& [depth, leaf_ver, script] : m_tap_tree) { + builder.Add((int)depth, script, (int)leaf_ver, /*track=*/true); + } + assert(builder.IsComplete()); + builder.Finalize(m_tap_internal_key); + TaprootSpendData spenddata = builder.GetSpendData(); sigdata.tr_spenddata.internal_key = m_tap_internal_key; sigdata.tr_spenddata.Merge(spenddata); @@ -243,8 +250,8 @@ void PSBTOutput::FromSignatureData(const SignatureData& sigdata) if (!sigdata.tr_spenddata.internal_key.IsNull()) { m_tap_internal_key = sigdata.tr_spenddata.internal_key; } - if (sigdata.tr_builder.has_value()) { - m_tap_tree = sigdata.tr_builder; + if (sigdata.tr_builder.has_value() && sigdata.tr_builder->HasScripts()) { + m_tap_tree = sigdata.tr_builder->GetTreeTuples(); } for (const auto& [pubkey, leaf_origin] : sigdata.taproot_misc_pubkeys) { m_tap_bip32_paths.emplace(pubkey, leaf_origin); @@ -265,13 +272,43 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; if (m_tap_internal_key.IsNull() && !output.m_tap_internal_key.IsNull()) m_tap_internal_key = output.m_tap_internal_key; - if (m_tap_tree.has_value() && !output.m_tap_tree.has_value()) m_tap_tree = output.m_tap_tree; + if (m_tap_tree.empty() && !output.m_tap_tree.empty()) m_tap_tree = output.m_tap_tree; } + bool PSBTInputSigned(const PSBTInput& input) { return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } +bool PSBTInputSignedAndVerified(const PartiallySignedTransaction psbt, unsigned int input_index, const PrecomputedTransactionData* txdata) +{ + CTxOut utxo; + assert(psbt.inputs.size() >= input_index); + const PSBTInput& input = psbt.inputs[input_index]; + + if (input.non_witness_utxo) { + // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. + COutPoint prevout = psbt.tx->vin[input_index].prevout; + if (prevout.n >= input.non_witness_utxo->vout.size()) { + return false; + } + if (input.non_witness_utxo->GetHash() != prevout.hash) { + return false; + } + utxo = input.non_witness_utxo->vout[prevout.n]; + } else if (!input.witness_utxo.IsNull()) { + utxo = input.witness_utxo; + } else { + return false; + } + + if (txdata) { + return VerifyScript(input.final_script_sig, utxo.scriptPubKey, &input.final_script_witness, STANDARD_SCRIPT_VERIFY_FLAGS, MutableTransactionSignatureChecker{&(*psbt.tx), input_index, utxo.nValue, *txdata, MissingDataBehavior::FAIL}); + } else { + return VerifyScript(input.final_script_sig, utxo.scriptPubKey, &input.final_script_witness, STANDARD_SCRIPT_VERIFY_FLAGS, MutableTransactionSignatureChecker{&(*psbt.tx), input_index, utxo.nValue, MissingDataBehavior::FAIL}); + } +} + size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt) { size_t count = 0; for (const auto& input : psbt.inputs) { @@ -325,7 +362,7 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& PSBTInput& input = psbt.inputs.at(index); const CMutableTransaction& tx = *psbt.tx; - if (PSBTInputSigned(input)) { + if (PSBTInputSignedAndVerified(psbt, index, txdata)) { return true; } diff --git a/src/psbt.h b/src/psbt.h index d5c67802c7..37bf142366 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -713,7 +713,7 @@ struct PSBTOutput CScript witness_script; std::map<CPubKey, KeyOriginInfo> hd_keypaths; XOnlyPubKey m_tap_internal_key; - std::optional<TaprootBuilder> m_tap_tree; + std::vector<std::tuple<uint8_t, uint8_t, CScript>> m_tap_tree; std::map<XOnlyPubKey, std::pair<std::set<uint256>, KeyOriginInfo>> m_tap_bip32_paths; std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; std::set<PSBTProprietary> m_proprietary; @@ -754,15 +754,11 @@ struct PSBTOutput } // Write taproot tree - if (m_tap_tree.has_value()) { + if (!m_tap_tree.empty()) { SerializeToVector(s, PSBT_OUT_TAP_TREE); std::vector<unsigned char> value; CVectorWriter s_value(s.GetType(), s.GetVersion(), value, 0); - const auto& tuples = m_tap_tree->GetTreeTuples(); - for (const auto& tuple : tuples) { - uint8_t depth = std::get<0>(tuple); - uint8_t leaf_ver = std::get<1>(tuple); - CScript script = std::get<2>(tuple); + for (const auto& [depth, leaf_ver, script] : m_tap_tree) { s_value << depth; s_value << leaf_ver; s_value << script; @@ -858,10 +854,13 @@ struct PSBTOutput } else if (key.size() != 1) { throw std::ios_base::failure("Output Taproot tree key is more than one byte type"); } - m_tap_tree.emplace(); std::vector<unsigned char> tree_v; s >> tree_v; SpanReader s_tree(s.GetType(), s.GetVersion(), tree_v); + if (s_tree.empty()) { + throw std::ios_base::failure("Output Taproot tree must not be empty"); + } + TaprootBuilder builder; while (!s_tree.empty()) { uint8_t depth; uint8_t leaf_ver; @@ -875,9 +874,10 @@ struct PSBTOutput if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); } - m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); + m_tap_tree.push_back(std::make_tuple(depth, leaf_ver, script)); + builder.Add((int)depth, script, (int)leaf_ver, true /* track */); } - if (!m_tap_tree->IsComplete()) { + if (!builder.IsComplete()) { throw std::ios_base::failure("Output Taproot tree is malformed"); } break; @@ -931,11 +931,6 @@ struct PSBTOutput } } - // Finalize m_tap_tree so that all of the computed things are computed - if (m_tap_tree.has_value() && m_tap_tree->IsComplete() && m_tap_internal_key.IsFullyValid()) { - m_tap_tree->Finalize(m_tap_internal_key); - } - if (!found_sep) { throw std::ios_base::failure("Separator is missing at the end of an output map"); } @@ -1223,9 +1218,12 @@ std::string PSBTRoleName(PSBTRole role); /** Compute a PrecomputedTransactionData object from a psbt. */ PrecomputedTransactionData PrecomputePSBTData(const PartiallySignedTransaction& psbt); -/** Checks whether a PSBTInput is already signed. */ +/** Checks whether a PSBTInput is already signed by checking for non-null finalized fields. */ bool PSBTInputSigned(const PSBTInput& input); +/** Checks whether a PSBTInput is already signed by doing script verification using final fields. */ +bool PSBTInputSignedAndVerified(const PartiallySignedTransaction psbt, unsigned int input_index, const PrecomputedTransactionData* txdata); + /** Signs a PSBTInput, verifying that all provided data matches what is being signed. * * txdata should be the output of PrecomputePSBTData (which can be shared across diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 582f02132a..6be47e94fe 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -286,7 +286,7 @@ <item> <widget class="QLineEdit" name="externalSignerPath"> <property name="toolTip"> - <string>Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> + <string>Full path to a %1 compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> </property> </widget> </item> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 2b6711ca40..7c91eac37e 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -91,7 +91,9 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->thirdPartyTxUrls->setVisible(false); } -#ifndef ENABLE_EXTERNAL_SIGNER +#ifdef ENABLE_EXTERNAL_SIGNER + ui->externalSignerPath->setToolTip(ui->externalSignerPath->toolTip().arg(PACKAGE_NAME)); +#else //: "External signing" means using devices such as hardware wallets. ui->externalSignerPath->setToolTip(tr("Compiled without external signing support (required for external signing)")); ui->externalSignerPath->setEnabled(false); diff --git a/src/rest.cpp b/src/rest.cpp index 7f00db2222..a10d8a433f 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -590,6 +590,48 @@ static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std: } } + +RPCHelpMan getdeploymentinfo(); + +static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) +{ + if (!CheckWarmup(req)) return false; + + std::string hash_str; + const RESTResponseFormat rf = ParseDataFormat(hash_str, str_uri_part); + + switch (rf) { + case RESTResponseFormat::JSON: { + JSONRPCRequest jsonRequest; + jsonRequest.context = context; + jsonRequest.params = UniValue(UniValue::VARR); + + if (!hash_str.empty()) { + uint256 hash; + if (!ParseHashStr(hash_str, hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hash_str); + } + + const ChainstateManager* chainman = GetChainman(context, req); + if (!chainman) return false; + if (!WITH_LOCK(::cs_main, return chainman->m_blockman.LookupBlockIndex(ParseHashV(hash_str, "blockhash")))) { + return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); + } + + jsonRequest.params.pushKV("blockhash", hash_str); + } + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, getdeploymentinfo().HandleRequest(jsonRequest).write() + "\n"); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + +} + static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) @@ -939,6 +981,8 @@ static const struct { {"/rest/mempool/", rest_mempool}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, + {"/rest/deploymentinfo/", rest_deploymentinfo}, + {"/rest/deploymentinfo", rest_deploymentinfo}, {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 93d11c8276..f8ba822f54 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1307,7 +1307,7 @@ UniValue DeploymentInfo(const CBlockIndex* blockindex, const ChainstateManager& } } // anon namespace -static RPCHelpMan getdeploymentinfo() +RPCHelpMan getdeploymentinfo() { return RPCHelpMan{"getdeploymentinfo", "Returns an object containing various state info regarding deployments of consensus changes.", @@ -2019,6 +2019,40 @@ public: } }; +static const auto scan_action_arg_desc = RPCArg{ + "action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" + "\"start\" for starting a scan\n" + "\"abort\" for aborting the current scan (returns true when abort was successful)\n" + "\"status\" for progress report (in %) of the current scan" +}; + +static const auto scan_objects_arg_desc = RPCArg{ + "scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" + "Every scan object is either a string descriptor or an object:", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"}, + }}, + }, + RPCArgOptions{.oneline_description="[scanobjects,...]"}, +}; + +static const auto scan_result_abort = RPCResult{ + "when action=='abort'", RPCResult::Type::BOOL, "success", + "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort" +}; +static const auto scan_result_status_none = RPCResult{ + "when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", "" +}; +static const auto scan_result_status_some = RPCResult{ + "when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", + {{RPCResult::Type::NUM, "progress", "Approximate percent complete"},} +}; + + static RPCHelpMan scantxoutset() { // scriptPubKey corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S @@ -2038,21 +2072,8 @@ static RPCHelpMan scantxoutset() "In the latter case, a range needs to be specified by below if different from 1000.\n" "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n", { - {"action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" - "\"start\" for starting a scan\n" - "\"abort\" for aborting the current scan (returns true when abort was successful)\n" - "\"status\" for progress report (in %) of the current scan"}, - {"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" - "Every scan object is either a string descriptor or an object:", - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", - { - {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, - {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"}, - }}, - }, - RPCArgOptions{.oneline_description="[scanobjects,...]"}}, + scan_action_arg_desc, + scan_objects_arg_desc, }, { RPCResult{"when action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", { @@ -2069,17 +2090,15 @@ static RPCHelpMan scantxoutset() {RPCResult::Type::STR_HEX, "scriptPubKey", "The script key"}, {RPCResult::Type::STR, "desc", "A specialized descriptor for the matched scriptPubKey"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the unspent output"}, + {RPCResult::Type::BOOL, "coinbase", "Whether this is a coinbase output"}, {RPCResult::Type::NUM, "height", "Height of the unspent transaction output"}, }}, }}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT}, }}, - RPCResult{"when action=='abort'", RPCResult::Type::BOOL, "success", "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort"}, - RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, - }}, - RPCResult{"when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", ""}, + scan_result_abort, + scan_result_status_some, + scan_result_status_none, }, RPCExamples{ HelpExampleCli("scantxoutset", "start \'[\"" + EXAMPLE_DESCRIPTOR_RAW + "\"]\'") + @@ -2172,6 +2191,7 @@ static RPCHelpMan scantxoutset() unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); + unspent.pushKV("coinbase", coin.IsCoinBase()); unspent.pushKV("height", (int32_t)coin.nHeight); unspents.push_back(unspent); @@ -2186,6 +2206,203 @@ static RPCHelpMan scantxoutset() }; } +/** RAII object to prevent concurrency issue when scanning blockfilters */ +static std::atomic<int> g_scanfilter_progress; +static std::atomic<int> g_scanfilter_progress_height; +static std::atomic<bool> g_scanfilter_in_progress; +static std::atomic<bool> g_scanfilter_should_abort_scan; +class BlockFiltersScanReserver +{ +private: + bool m_could_reserve{false}; +public: + explicit BlockFiltersScanReserver() = default; + + bool reserve() { + CHECK_NONFATAL(!m_could_reserve); + if (g_scanfilter_in_progress.exchange(true)) { + return false; + } + m_could_reserve = true; + return true; + } + + ~BlockFiltersScanReserver() { + if (m_could_reserve) { + g_scanfilter_in_progress = false; + } + } +}; + +static RPCHelpMan scanblocks() +{ + return RPCHelpMan{"scanblocks", + "\nReturn relevant blockhashes for given descriptors.\n" + "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", + { + scan_action_arg_desc, + scan_objects_arg_desc, + RPCArg{"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "Height to start to scan from"}, + RPCArg{"stop_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"chain tip"}, "Height to stop to scan"}, + RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC)}, "The type name of the filter"} + }, + { + scan_result_status_none, + RPCResult{"When action=='start'", RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "from_height", "The height we started the scan from"}, + {RPCResult::Type::NUM, "to_height", "The height we ended the scan at"}, + {RPCResult::Type::ARR, "relevant_blocks", "", {{RPCResult::Type::STR_HEX, "blockhash", "A relevant blockhash"},}}, + }, + }, + RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, + {RPCResult::Type::NUM, "current_height", "Height of the block currently being scanned"}, + }, + }, + scan_result_abort, + }, + RPCExamples{ + HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 300000") + + HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 100 150 basic") + + HelpExampleCli("scanblocks", "status") + + HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 300000") + + HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 100, 150, \"basic\"") + + HelpExampleRpc("scanblocks", "\"status\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue ret(UniValue::VOBJ); + if (request.params[0].get_str() == "status") { + BlockFiltersScanReserver reserver; + if (reserver.reserve()) { + // no scan in progress + return NullUniValue; + } + ret.pushKV("progress", g_scanfilter_progress.load()); + ret.pushKV("current_height", g_scanfilter_progress_height.load()); + return ret; + } else if (request.params[0].get_str() == "abort") { + BlockFiltersScanReserver reserver; + if (reserver.reserve()) { + // reserve was possible which means no scan was running + return false; + } + // set the abort flag + g_scanfilter_should_abort_scan = true; + return true; + } + else if (request.params[0].get_str() == "start") { + BlockFiltersScanReserver reserver; + if (!reserver.reserve()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); + } + const std::string filtertype_name{request.params[4].isNull() ? "basic" : request.params[4].get_str()}; + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(filtertype_name, filtertype)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype"); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = EnsureChainman(node); + + // set the start-height + const CBlockIndex* block = nullptr; + const CBlockIndex* stop_block = nullptr; + { + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + block = active_chain.Genesis(); + stop_block = active_chain.Tip(); + if (!request.params[2].isNull()) { + block = active_chain[request.params[2].getInt<int>()]; + if (!block) { + throw JSONRPCError(RPC_MISC_ERROR, "Invalid start_height"); + } + } + if (!request.params[3].isNull()) { + stop_block = active_chain[request.params[3].getInt<int>()]; + if (!stop_block || stop_block->nHeight < block->nHeight) { + throw JSONRPCError(RPC_MISC_ERROR, "Invalid stop_height"); + } + } + } + CHECK_NONFATAL(block); + + // loop through the scan objects, add scripts to the needle_set + GCSFilter::ElementSet needle_set; + for (const UniValue& scanobject : request.params[1].get_array().getValues()) { + FlatSigningProvider provider; + std::vector<CScript> scripts = EvalDescriptorStringOrObject(scanobject, provider); + for (const CScript& script : scripts) { + needle_set.emplace(script.begin(), script.end()); + } + } + UniValue blocks(UniValue::VARR); + const int amount_per_chunk = 10000; + const CBlockIndex* start_index = block; // for remembering the start of a blockfilter range + std::vector<BlockFilter> filters; + const CBlockIndex* start_block = block; // for progress reporting + const int total_blocks_to_process = stop_block->nHeight - start_block->nHeight; + + g_scanfilter_should_abort_scan = false; + g_scanfilter_progress = 0; + g_scanfilter_progress_height = start_block->nHeight; + + while (block) { + node.rpc_interruption_point(); // allow a clean shutdown + if (g_scanfilter_should_abort_scan) { + LogPrintf("scanblocks RPC aborted at height %d.\n", block->nHeight); + break; + } + const CBlockIndex* next = nullptr; + { + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + next = active_chain.Next(block); + if (block == stop_block) next = nullptr; + } + if (start_index->nHeight + amount_per_chunk == block->nHeight || next == nullptr) { + LogPrint(BCLog::RPC, "Fetching blockfilters from height %d to height %d.\n", start_index->nHeight, block->nHeight); + if (index->LookupFilterRange(start_index->nHeight, block, filters)) { + for (const BlockFilter& filter : filters) { + // compare the elements-set with each filter + if (filter.GetFilter().MatchAny(needle_set)) { + blocks.push_back(filter.GetBlockHash().GetHex()); + LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); + } + } + } + start_index = block; + + // update progress + int blocks_processed = block->nHeight - start_block->nHeight; + if (total_blocks_to_process > 0) { // avoid division by zero + g_scanfilter_progress = (int)(100.0 / total_blocks_to_process * blocks_processed); + } else { + g_scanfilter_progress = 100; + } + g_scanfilter_progress_height = block->nHeight; + } + block = next; + } + ret.pushKV("from_height", start_block->nHeight); + ret.pushKV("to_height", g_scanfilter_progress_height.load()); + ret.pushKV("relevant_blocks", blocks); + } + else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid command"); + } + return ret; +}, + }; +} + static RPCHelpMan getblockfilter() { return RPCHelpMan{"getblockfilter", @@ -2421,6 +2638,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &verifychain}, {"blockchain", &preciousblock}, {"blockchain", &scantxoutset}, + {"blockchain", &scanblocks}, {"blockchain", &getblockfilter}, {"hidden", &invalidateblock}, {"hidden", &reconsiderblock}, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 612dbbdacf..8688263ef5 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -83,6 +83,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 8, "fee_rate"}, { "sendmany", 9, "verbose" }, { "deriveaddresses", 1, "range" }, + { "scanblocks", 1, "scanobjects" }, + { "scanblocks", 2, "start_height" }, + { "scanblocks", 3, "stop_height" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index e390a7c15c..706d783942 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -449,9 +449,8 @@ static RPCHelpMan getmempoolancestors() } CTxMemPool::setEntries setAncestors; - uint64_t noLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - mempool.CalculateMemPoolAncestors(*it, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); + mempool.CalculateMemPoolAncestors(*it, setAncestors, CTxMemPool::Limits::NoLimits(), dummy, false); if (!fVerbose) { UniValue o(UniValue::VARR); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index c4bd03fa9f..98383fdaca 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -730,10 +730,10 @@ static RPCHelpMan getblocktemplate() // Update block static CBlockIndex* pindexPrev; - static int64_t nStart; + static int64_t time_start; static std::unique_ptr<CBlockTemplate> pblocktemplate; if (pindexPrev != active_chain.Tip() || - (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - nStart > 5)) + (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLast && GetTime() - time_start > 5)) { // Clear pindexPrev so future calls make a new block, despite any failures from here on pindexPrev = nullptr; @@ -741,7 +741,7 @@ static RPCHelpMan getblocktemplate() // Store the pindexBest used before CreateNewBlock, to avoid races nTransactionsUpdatedLast = mempool.GetTransactionsUpdated(); CBlockIndex* pindexPrevNew = active_chain.Tip(); - nStart = GetTime(); + time_start = GetTime(); // Create new block CScript scriptDummy = CScript() << OP_TRUE; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index d701a180ab..8d7f4e7f5b 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -114,7 +114,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether peer has asked us to relay transactions to it"}, + {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether we relay transactions to this peer"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, @@ -123,9 +123,9 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, - {RPCResult::Type::NUM, "pingtime", /*optional=*/true, "ping time (if available)"}, - {RPCResult::Type::NUM, "minping", /*optional=*/true, "minimum observed ping time (if any at all)"}, - {RPCResult::Type::NUM, "pingwait", /*optional=*/true, "ping wait (if non-zero)"}, + {RPCResult::Type::NUM, "pingtime", /*optional=*/true, "The last ping time in milliseconds (ms), if any"}, + {RPCResult::Type::NUM, "minping", /*optional=*/true, "The minimum observed ping time in milliseconds (ms), if any"}, + {RPCResult::Type::NUM, "pingwait", /*optional=*/true, "The duration in milliseconds (ms) of an outstanding ping (if non-zero)"}, {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, {RPCResult::Type::STR, "subver", "The string version"}, {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, @@ -146,7 +146,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, }}, - {RPCResult::Type::NUM, "minfeefilter", /*optional=*/true, "The minimum fee rate for transactions this peer accepts"}, + {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", { {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" @@ -200,6 +200,9 @@ static RPCHelpMan getpeerinfo() ServiceFlags services{fStateStats ? statestats.their_services : ServiceFlags::NODE_NONE}; obj.pushKV("services", strprintf("%016x", services)); obj.pushKV("servicesnames", GetServicesNames(services)); + if (fStateStats) { + obj.pushKV("relaytxes", statestats.m_relay_txs); + } obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -235,8 +238,6 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); - obj.pushKV("relaytxes", statestats.m_relay_txs); - obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); obj.pushKV("addr_processed", statestats.m_addr_processed); obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); @@ -246,6 +247,7 @@ static RPCHelpMan getpeerinfo() permissions.push_back(permission); } obj.pushKV("permissions", permissions); + obj.pushKV("minfeefilter", fStateStats ? ValueFromAmount(statestats.m_fee_filter_received) : 0); UniValue sendPerMsgType(UniValue::VOBJ); for (const auto& i : stats.mapSendBytesPerMsgType) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index f365de7d0c..d654de1862 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1241,13 +1241,9 @@ static RPCHelpMan decodepsbt() } // Taproot tree - if (output.m_tap_tree.has_value()) { + if (!output.m_tap_tree.empty()) { UniValue tree(UniValue::VARR); - const auto& tuples = output.m_tap_tree->GetTreeTuples(); - for (const auto& tuple : tuples) { - uint8_t depth = std::get<0>(tuple); - uint8_t leaf_ver = std::get<1>(tuple); - CScript script = std::get<2>(tuple); + for (const auto& [depth, leaf_ver, script] : output.m_tap_tree) { UniValue elem(UniValue::VOBJ); elem.pushKV("depth", (int)depth); elem.pushKV("leaf_ver", (int)leaf_ver); diff --git a/src/script/standard.h b/src/script/standard.h index 1e6769782a..966a52b2c7 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -315,6 +315,8 @@ public: TaprootSpendData GetSpendData() const; /** Returns a vector of tuples representing the depth, leaf version, and script */ std::vector<std::tuple<uint8_t, uint8_t, CScript>> GetTreeTuples() const; + /** Returns true if there are any tapscripts */ + bool HasScripts() const { return !m_branch.empty(); } }; /** Given a TaprootSpendData and the output key, reconstruct its script tree. diff --git a/src/streams.h b/src/streams.h index 24778ab331..0178df1c49 100644 --- a/src/streams.h +++ b/src/streams.h @@ -269,7 +269,6 @@ public: // Stream subset // bool eof() const { return size() == 0; } - CDataStream* rdbuf() { return this; } int in_avail() const { return size(); } void SetType(int n) { nType = n; } @@ -488,12 +487,14 @@ public: AutoFile(const AutoFile&) = delete; AutoFile& operator=(const AutoFile&) = delete; - void fclose() + int fclose() { + int retval{0}; if (file) { - ::fclose(file); + retval = ::fclose(file); file = nullptr; } + return retval; } /** Get wrapped FILE* with transfer of ownership. diff --git a/src/sync.h b/src/sync.h index a9d0091bd2..1f4e191214 100644 --- a/src/sync.h +++ b/src/sync.h @@ -111,7 +111,7 @@ public: return PARENT::try_lock(); } - using UniqueLock = std::unique_lock<PARENT>; + using unique_lock = std::unique_lock<PARENT>; #ifdef __clang__ //! For negative capabilities in the Clang Thread Safety Analysis. //! A negative requirement uses the EXCLUSIVE_LOCKS_REQUIRED attribute, in conjunction @@ -147,11 +147,13 @@ inline void AssertLockNotHeldInline(const char* name, const char* file, int line inline void AssertLockNotHeldInline(const char* name, const char* file, int line, GlobalMutex* cs) LOCKS_EXCLUDED(cs) { AssertLockNotHeldInternal(name, file, line, cs); } #define AssertLockNotHeld(cs) AssertLockNotHeldInline(#cs, __FILE__, __LINE__, &cs) -/** Wrapper around std::unique_lock style lock for Mutex. */ -template <typename Mutex, typename Base = typename Mutex::UniqueLock> -class SCOPED_LOCKABLE UniqueLock : public Base +/** Wrapper around std::unique_lock style lock for MutexType. */ +template <typename MutexType> +class SCOPED_LOCKABLE UniqueLock : public MutexType::unique_lock { private: + using Base = typename MutexType::unique_lock; + void Enter(const char* pszName, const char* pszFile, int nLine) { EnterCritical(pszName, pszFile, nLine, Base::mutex()); @@ -173,7 +175,7 @@ private: } public: - UniqueLock(Mutex& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : Base(mutexIn, std::defer_lock) + UniqueLock(MutexType& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : Base(mutexIn, std::defer_lock) { if (fTry) TryEnter(pszName, pszFile, nLine); @@ -181,7 +183,7 @@ public: Enter(pszName, pszFile, nLine); } - UniqueLock(Mutex* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) + UniqueLock(MutexType* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) { if (!pmutexIn) return; @@ -241,29 +243,24 @@ public: #define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock UNIQUE_NAME(revlock)(g, #g, __FILE__, __LINE__) -template<typename MutexArg> -using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; - // When locking a Mutex, require negative capability to ensure the lock // is not already held inline Mutex& MaybeCheckNotHeld(Mutex& cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) LOCK_RETURNED(cs) { return cs; } inline Mutex* MaybeCheckNotHeld(Mutex* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) LOCK_RETURNED(cs) { return cs; } -// When locking a GlobalMutex, just check it is not locked in the surrounding scope -inline GlobalMutex& MaybeCheckNotHeld(GlobalMutex& cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } -inline GlobalMutex* MaybeCheckNotHeld(GlobalMutex* cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } - -// When locking a RecursiveMutex, it's okay to already hold the lock -// but check that it is not known to be locked in the surrounding scope anyway -inline RecursiveMutex& MaybeCheckNotHeld(RecursiveMutex& cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } -inline RecursiveMutex* MaybeCheckNotHeld(RecursiveMutex* cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } +// When locking a GlobalMutex or RecursiveMutex, just check it is not +// locked in the surrounding scope. +template <typename MutexType> +inline MutexType& MaybeCheckNotHeld(MutexType& m) LOCKS_EXCLUDED(m) LOCK_RETURNED(m) { return m; } +template <typename MutexType> +inline MutexType* MaybeCheckNotHeld(MutexType* m) LOCKS_EXCLUDED(m) LOCK_RETURNED(m) { return m; } -#define LOCK(cs) DebugLock<decltype(cs)> UNIQUE_NAME(criticalblock)(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) +#define LOCK(cs) UniqueLock UNIQUE_NAME(criticalblock)(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) #define LOCK2(cs1, cs2) \ - DebugLock<decltype(cs1)> criticalblock1(MaybeCheckNotHeld(cs1), #cs1, __FILE__, __LINE__); \ - DebugLock<decltype(cs2)> criticalblock2(MaybeCheckNotHeld(cs2), #cs2, __FILE__, __LINE__) -#define TRY_LOCK(cs, name) DebugLock<decltype(cs)> name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__, true) -#define WAIT_LOCK(cs, name) DebugLock<decltype(cs)> name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) + UniqueLock criticalblock1(MaybeCheckNotHeld(cs1), #cs1, __FILE__, __LINE__); \ + UniqueLock criticalblock2(MaybeCheckNotHeld(cs2), #cs2, __FILE__, __LINE__) +#define TRY_LOCK(cs, name) UniqueLock name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__, true) +#define WAIT_LOCK(cs, name) UniqueLock name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) #define ENTER_CRITICAL_SECTION(cs) \ { \ diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 78b82b9b20..1c13c0a909 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/merkle.h> #include <pow.h> #include <streams.h> +#include <test/util/txmempool.h> #include <test/util/setup_common.h> diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp new file mode 100644 index 0000000000..dd7c32cc46 --- /dev/null +++ b/src/test/blockmanager_tests.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <node/blockstorage.h> +#include <node/context.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> + +using node::BlockManager; +using node::BLOCK_SERIALIZATION_HEADER_SIZE; + +// use BasicTestingSetup here for the data directory configuration, setup, and cleanup +BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) +{ + const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; + BlockManager blockman {}; + CChain chain {}; + // simulate adding a genesis block normally + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // simulate what happens during reindex + // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file + // the block is found at offset 8 because there is an 8 byte serialization header + // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. + FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // now simulate what happens after reindex for the first new block processed + // the actual block contents don't matter, just that it's a block. + // verify that the write position is at offset 0x12d. + // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur + // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 + // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 2a6a777cfe..8a2b0792fd 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -76,10 +76,16 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_CHECK(block_index != new_block_index); + // It is not safe to stop and destroy the index until it finishes handling + // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain() + // call above is sufficient to ensure this, but the + // SyncWithValidationInterfaceQueue() call below is also needed to ensure + // TSAN always sees the test thread waiting for the notification thread, and + // avoid potential false positive reports. + SyncWithValidationInterfaceQueue(); + // Shutdown sequence (c.f. Shutdown() in init.cpp) coin_stats_index.Stop(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() } // Test shutdown between BlockConnected and ChainStateFlushed notifications, diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index c52fca5fe8..f05248ab47 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -12,6 +12,7 @@ #include <key_io.h> #include <memusage.h> #include <netbase.h> +#include <policy/policy.h> #include <policy/settings.h> #include <pow.h> #include <protocol.h> diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 637ba503c6..a3d57dbdd5 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -8,6 +8,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> #include <txmempool.h> diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index f6a35da4fc..5a4df735da 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -130,6 +130,7 @@ FUZZ_TARGET_MSG(pong); FUZZ_TARGET_MSG(sendaddrv2); FUZZ_TARGET_MSG(sendcmpct); FUZZ_TARGET_MSG(sendheaders); +FUZZ_TARGET_MSG(sendtxrcncl); FUZZ_TARGET_MSG(tx); FUZZ_TARGET_MSG(verack); FUZZ_TARGET_MSG(version); diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 1a06ae886e..678fc7a5aa 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -9,7 +9,9 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <cstdint> diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 26913a41d2..f32046e69f 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -151,6 +151,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "preciousblock", "pruneblockchain", "reconsiderblock", + "scanblocks", "scantxoutset", "sendrawtransaction", "setmocktime", diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 283a146369..46ca8e47e9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -8,11 +8,12 @@ #include <node/miner.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> #include <test/util/mining.h> #include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <util/rbf.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 8625898855..d495a6bfe3 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -16,7 +16,7 @@ #include <memory> FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) - : m_fuzzed_data_provider{fuzzed_data_provider} + : m_fuzzed_data_provider{fuzzed_data_provider}, m_selectable{fuzzed_data_provider.ConsumeBool()} { m_socket = fuzzed_data_provider.ConsumeIntegralInRange<SOCKET>(INVALID_SOCKET - 1, INVALID_SOCKET); } @@ -254,6 +254,24 @@ int FuzzedSock::GetSockName(sockaddr* name, socklen_t* name_len) const return 0; } +bool FuzzedSock::SetNonBlocking() const +{ + constexpr std::array setnonblocking_errnos{ + EBADF, + EPERM, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, setnonblocking_errnos); + return false; + } + return true; +} + +bool FuzzedSock::IsSelectable() const +{ + return m_selectable; +} + bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { constexpr std::array wait_errnos{ @@ -478,21 +496,6 @@ CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) no return tx_destination; } -CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept -{ - // Avoid: - // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' - // - // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() - const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); - assert(MoneyRange(fee)); - const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); - const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); - return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; -} - bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept { for (const CTxIn& tx_in : tx.vin) { diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 36d55079cb..dfe4855326 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -23,7 +23,6 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/util/net.h> -#include <txmempool.h> #include <uint256.h> #include <version.h> @@ -48,6 +47,13 @@ class FuzzedSock : public Sock */ mutable std::optional<uint8_t> m_peek_data; + /** + * Whether to pretend that the socket is select(2)-able. This is randomly set in the + * constructor. It should remain constant so that repeated calls to `IsSelectable()` + * return the same value. + */ + const bool m_selectable; + public: explicit FuzzedSock(FuzzedDataProvider& fuzzed_data_provider); @@ -73,6 +79,10 @@ public: int GetSockName(sockaddr* name, socklen_t* name_len) const override; + bool SetNonBlocking() const override; + + bool IsSelectable() const override; + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; bool WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const override; @@ -213,8 +223,6 @@ template <typename WeakEnumType, size_t size> return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } -[[nodiscard]] CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept; - [[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; template <typename T> diff --git a/src/test/fuzz/util/mempool.cpp b/src/test/fuzz/util/mempool.cpp new file mode 100644 index 0000000000..d0053f77d2 --- /dev/null +++ b/src/test/fuzz/util/mempool.cpp @@ -0,0 +1,27 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/amount.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> +#include <txmempool.h> + +#include <limits> + +CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept +{ + // Avoid: + // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' + // + // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/std::numeric_limits<CAmount>::max() / CAmount{100'000})}; + assert(MoneyRange(fee)); + const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); + const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; +} diff --git a/src/test/fuzz/mempool_utils.h b/src/test/fuzz/util/mempool.h index c172e8c4b7..4304e5294e 100644 --- a/src/test/fuzz/mempool_utils.h +++ b/src/test/fuzz/util/mempool.h @@ -2,9 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H -#define BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H +#ifndef BITCOIN_TEST_FUZZ_UTIL_MEMPOOL_H +#define BITCOIN_TEST_FUZZ_UTIL_MEMPOOL_H +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <txmempool.h> #include <validation.h> class DummyChainState final : public Chainstate @@ -16,4 +19,6 @@ public: } }; -#endif // BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H +[[nodiscard]] CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept; + +#endif // BITCOIN_TEST_FUZZ_UTIL_MEMPOOL_H diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 8241dff189..9a90de8911 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -9,9 +9,10 @@ #include <node/mempool_persist_args.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/time.h> #include <validation.h> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 8c745b07b9..393311e4e4 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/policy.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/system.h> #include <util/time.h> @@ -203,7 +204,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CTxMemPool::setEntries setAncestorsCalculated; std::string dummy; - BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry.Fee(2000000LL).FromTx(tx7), setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); + BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry.Fee(2'000'000LL).FromTx(tx7), setAncestorsCalculated, CTxMemPool::Limits::NoLimits(), dummy), true); BOOST_CHECK(setAncestorsCalculated == setAncestors); pool.addUnchecked(entry.FromTx(tx7), setAncestors); @@ -261,7 +262,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx10.vout[0].nValue = 10 * COIN; setAncestorsCalculated.clear(); - BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry.Fee(200000LL).Time(4).FromTx(tx10), setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); + BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry.Fee(200'000LL).Time(4).FromTx(tx10), setAncestorsCalculated, CTxMemPool::Limits::NoLimits(), dummy), true); BOOST_CHECK(setAncestorsCalculated == setAncestors); pool.addUnchecked(entry.FromTx(tx10), setAncestors); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 9f5fb17b60..9bc29e3599 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparams.h> #include <coins.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -10,6 +9,7 @@ #include <node/miner.h> #include <policy/policy.h> #include <script/standard.h> +#include <test/util/txmempool.h> #include <timedata.h> #include <txmempool.h> #include <uint256.h> @@ -30,15 +30,24 @@ using node::CBlockTemplate; namespace miner_tests { struct MinerTestingSetup : public TestingSetup { - void TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); - void TestBasicMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); - void TestPrioritisedMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs); - bool TestSequenceLocks(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_node.mempool->cs) + void TestPackageSelection(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void TestBasicMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void TestPrioritisedMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool TestSequenceLocks(const CTransaction& tx, CTxMemPool& tx_mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - CCoinsViewMemPool view_mempool(&m_node.chainman->ActiveChainstate().CoinsTip(), *m_node.mempool); + CCoinsViewMemPool view_mempool{&m_node.chainman->ActiveChainstate().CoinsTip(), tx_mempool}; return CheckSequenceLocksAtTip(m_node.chainman->ActiveChain().Tip(), view_mempool, tx); } - BlockAssembler AssemblerForTest(const CChainParams& params); + CTxMemPool& MakeMempool() + { + // Delete the previous mempool to ensure with valgrind that the old + // pointer is not accessed, when the new one should be accessed + // instead. + m_node.mempool.reset(); + m_node.mempool = std::make_unique<CTxMemPool>(MemPoolOptionsForTest(m_node)); + return *m_node.mempool; + } + BlockAssembler AssemblerForTest(CTxMemPool& tx_mempool); }; } // namespace miner_tests @@ -46,13 +55,13 @@ BOOST_FIXTURE_TEST_SUITE(miner_tests, MinerTestingSetup) static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); -BlockAssembler MinerTestingSetup::AssemblerForTest(const CChainParams& params) +BlockAssembler MinerTestingSetup::AssemblerForTest(CTxMemPool& tx_mempool) { BlockAssembler::Options options; options.nBlockMaxWeight = MAX_BLOCK_WEIGHT; options.blockMinFeeRate = blockMinFeeRate; - return BlockAssembler{m_node.chainman->ActiveChainstate(), m_node.mempool.get(), options}; + return BlockAssembler{m_node.chainman->ActiveChainstate(), &tx_mempool, options}; } constexpr static struct { @@ -89,8 +98,10 @@ static CBlockIndex CreateBlockIndex(int nHeight, CBlockIndex* active_chain_tip) // Test suite for ancestor feerate transaction selection. // Implemented as an additional function, rather than a separate test case, // to allow reusing the blockchain created in CreateNewBlock_validity. -void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) +void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); // Test the ancestor feerate transaction selection. TestMemPoolEntryHelper entry; @@ -105,21 +116,21 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis uint256 hashParentTx = tx.GetHash(); // save this txid for later use - m_node.mempool->addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; uint256 hashMediumFeeTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); // This tx has a high fee, but depends on the first transaction tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee uint256 hashHighFeeTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 4U); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); @@ -129,7 +140,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vin[0].prevout.hash = hashHighFeeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee uint256 hashFreeTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(0).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(0).FromTx(tx)); size_t freeTxSize = ::GetSerializeSize(tx, PROTOCOL_VERSION); // Calculate a fee on child transaction that will put the package just @@ -139,8 +150,8 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vin[0].prevout.hash = hashFreeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse; uint256 hashLowFeeTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(feeToUse).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + tx_mempool.addUnchecked(entry.Fee(feeToUse).FromTx(tx)); + pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); // Verify that the free tx and the low fee tx didn't get selected for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { BOOST_CHECK(pblocktemplate->block.vtx[i]->GetHash() != hashFreeTx); @@ -150,11 +161,11 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co // Test that packages above the min relay fee do get included, even if one // of the transactions is below the min relay fee // Remove the low fee transaction and replace with a higher fee transaction - m_node.mempool->removeRecursive(CTransaction(tx), MemPoolRemovalReason::REPLACED); + tx_mempool.removeRecursive(CTransaction(tx), MemPoolRemovalReason::REPLACED); tx.vout[0].nValue -= 2; // Now we should be just over the min relay fee hashLowFeeTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(feeToUse+2).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + tx_mempool.addUnchecked(entry.Fee(feeToUse + 2).FromTx(tx)); + pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); @@ -167,7 +178,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue = 5000000000LL - 100000000; tx.vout[1].nValue = 100000000; // 1BTC output uint256 hashFreeTx2 = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); // This tx can't be mined by itself tx.vin[0].prevout.hash = hashFreeTx2; @@ -175,8 +186,8 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co feeToUse = blockMinFeeRate.GetFee(freeTxSize); tx.vout[0].nValue = 5000000000LL - 100000000 - feeToUse; uint256 hashLowFeeTx2 = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + tx_mempool.addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); + pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); // Verify that this tx isn't selected. for (size_t i=0; i<pblocktemplate->block.vtx.size(); ++i) { @@ -188,13 +199,13 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co // as well. tx.vin[0].prevout.n = 1; tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee - m_node.mempool->addUnchecked(entry.Fee(10000).FromTx(tx)); - pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + tx_mempool.addUnchecked(entry.Fee(10000).FromTx(tx)); + pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 9U); BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } -void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) +void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) { uint256 hash; CMutableTransaction tx; @@ -202,172 +213,205 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C entry.nFee = 11; entry.nHeight = 11; - // Just to make sure we can still make simple blocks - auto pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); - BOOST_CHECK(pblocktemplate); - - const CAmount BLOCKSUBSIDY = 50*COIN; + const CAmount BLOCKSUBSIDY = 50 * COIN; const CAmount LOWFEE = CENT; const CAmount HIGHFEE = COIN; - const CAmount HIGHERFEE = 4*COIN; + const CAmount HIGHERFEE = 4 * COIN; - // block sigops > limit: 1000 CHECKMULTISIG + 1 - tx.vin.resize(1); - // NOTE: OP_NOP is used to force 20 SigOps for the CHECKMULTISIG - tx.vin[0].scriptSig = CScript() << OP_0 << OP_0 << OP_0 << OP_NOP << OP_CHECKMULTISIG << OP_1; - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vin[0].prevout.n = 0; - tx.vout.resize(1); - tx.vout[0].nValue = BLOCKSUBSIDY; - for (unsigned int i = 0; i < 1001; ++i) { - tx.vout[0].nValue -= LOWFEE; - hash = tx.GetHash(); - bool spendsCoinbase = i == 0; // only first tx spends coinbase - // If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); - tx.vin[0].prevout.hash = hash; + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + // Just to make sure we can still make simple blocks + auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); + BOOST_CHECK(pblocktemplate); + + // block sigops > limit: 1000 CHECKMULTISIG + 1 + tx.vin.resize(1); + // NOTE: OP_NOP is used to force 20 SigOps for the CHECKMULTISIG + tx.vin[0].scriptSig = CScript() << OP_0 << OP_0 << OP_0 << OP_NOP << OP_CHECKMULTISIG << OP_1; + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vin[0].prevout.n = 0; + tx.vout.resize(1); + tx.vout[0].nValue = BLOCKSUBSIDY; + for (unsigned int i = 0; i < 1001; ++i) { + tx.vout[0].nValue -= LOWFEE; + hash = tx.GetHash(); + bool spendsCoinbase = i == 0; // only first tx spends coinbase + // If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + } + + BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); } - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); - m_node.mempool->clear(); + { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vout[0].nValue = BLOCKSUBSIDY; + for (unsigned int i = 0; i < 1001; ++i) { + tx.vout[0].nValue -= LOWFEE; + hash = tx.GetHash(); + bool spendsCoinbase = i == 0; // only first tx spends coinbase + // If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + } + BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + } - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vout[0].nValue = BLOCKSUBSIDY; - for (unsigned int i = 0; i < 1001; ++i) { - tx.vout[0].nValue -= LOWFEE; - hash = tx.GetHash(); - bool spendsCoinbase = i == 0; // only first tx spends coinbase - // If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); - tx.vin[0].prevout.hash = hash; + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + // block size > limit + tx.vin[0].scriptSig = CScript(); + // 18 * (520char + DROP) + OP_1 = 9433 bytes + std::vector<unsigned char> vchData(520); + for (unsigned int i = 0; i < 18; ++i) { + tx.vin[0].scriptSig << vchData << OP_DROP; + } + tx.vin[0].scriptSig << OP_1; + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vout[0].nValue = BLOCKSUBSIDY; + for (unsigned int i = 0; i < 128; ++i) { + tx.vout[0].nValue -= LOWFEE; + hash = tx.GetHash(); + bool spendsCoinbase = i == 0; // only first tx spends coinbase + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + } + BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - m_node.mempool->clear(); - - // block size > limit - tx.vin[0].scriptSig = CScript(); - // 18 * (520char + DROP) + OP_1 = 9433 bytes - std::vector<unsigned char> vchData(520); - for (unsigned int i = 0; i < 18; ++i) - tx.vin[0].scriptSig << vchData << OP_DROP; - tx.vin[0].scriptSig << OP_1; - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vout[0].nValue = BLOCKSUBSIDY; - for (unsigned int i = 0; i < 128; ++i) + { - tx.vout[0].nValue -= LOWFEE; + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + // orphan in tx_mempool, template creation fails hash = tx.GetHash(); - bool spendsCoinbase = i == 0; // only first tx spends coinbase - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); - tx.vin[0].prevout.hash = hash; + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); + BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - m_node.mempool->clear(); - - // orphan in *m_node.mempool, template creation fails - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); - m_node.mempool->clear(); - // child with higher feerate than parent - tx.vin[0].scriptSig = CScript() << OP_1; - tx.vin[0].prevout.hash = txFirst[1]->GetHash(); - tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - tx.vin[0].prevout.hash = hash; - tx.vin.resize(2); - tx.vin[1].scriptSig = CScript() << OP_1; - tx.vin[1].prevout.hash = txFirst[0]->GetHash(); - tx.vin[1].prevout.n = 0; - tx.vout[0].nValue = tx.vout[0].nValue+BLOCKSUBSIDY-HIGHERFEE; //First txn output + fresh coinbase - new txn fee - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - m_node.mempool->clear(); + { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); - // coinbase in *m_node.mempool, template creation fails - tx.vin.resize(1); - tx.vin[0].prevout.SetNull(); - tx.vin[0].scriptSig = CScript() << OP_0 << OP_1; - tx.vout[0].nValue = 0; - hash = tx.GetHash(); - // give it a fee so it'll get mined - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - // Should throw bad-cb-multiple - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); - m_node.mempool->clear(); + // child with higher feerate than parent + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vin[0].prevout.hash = txFirst[1]->GetHash(); + tx.vout[0].nValue = BLOCKSUBSIDY - HIGHFEE; + hash = tx.GetHash(); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + tx.vin.resize(2); + tx.vin[1].scriptSig = CScript() << OP_1; + tx.vin[1].prevout.hash = txFirst[0]->GetHash(); + tx.vin[1].prevout.n = 0; + tx.vout[0].nValue = tx.vout[0].nValue + BLOCKSUBSIDY - HIGHERFEE; // First txn output + fresh coinbase - new txn fee + hash = tx.GetHash(); + tx_mempool.addUnchecked(entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + } - // double spend txn pair in *m_node.mempool, template creation fails - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vin[0].scriptSig = CScript() << OP_1; - tx.vout[0].nValue = BLOCKSUBSIDY-HIGHFEE; - tx.vout[0].scriptPubKey = CScript() << OP_1; - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - tx.vout[0].scriptPubKey = CScript() << OP_2; - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); - m_node.mempool->clear(); - - // subsidy changing - int nHeight = m_node.chainman->ActiveChain().Height(); - // Create an actual 209999-long block chain (without valid blocks). - while (m_node.chainman->ActiveChain().Tip()->nHeight < 209999) { - CBlockIndex* prev = m_node.chainman->ActiveChain().Tip(); - CBlockIndex* next = new CBlockIndex(); - next->phashBlock = new uint256(InsecureRand256()); - m_node.chainman->ActiveChainstate().CoinsTip().SetBestBlock(next->GetBlockHash()); - next->pprev = prev; - next->nHeight = prev->nHeight + 1; - next->BuildSkip(); - m_node.chainman->ActiveChain().SetTip(*next); + { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + // coinbase in tx_mempool, template creation fails + tx.vin.resize(1); + tx.vin[0].prevout.SetNull(); + tx.vin[0].scriptSig = CScript() << OP_0 << OP_1; + tx.vout[0].nValue = 0; + hash = tx.GetHash(); + // give it a fee so it'll get mined + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + // Should throw bad-cb-multiple + BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - // Extend to a 210000-long block chain. - while (m_node.chainman->ActiveChain().Tip()->nHeight < 210000) { - CBlockIndex* prev = m_node.chainman->ActiveChain().Tip(); - CBlockIndex* next = new CBlockIndex(); - next->phashBlock = new uint256(InsecureRand256()); - m_node.chainman->ActiveChainstate().CoinsTip().SetBestBlock(next->GetBlockHash()); - next->pprev = prev; - next->nHeight = prev->nHeight + 1; - next->BuildSkip(); - m_node.chainman->ActiveChain().SetTip(*next); + + { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + // double spend txn pair in tx_mempool, template creation fails + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vout[0].nValue = BLOCKSUBSIDY - HIGHFEE; + tx.vout[0].scriptPubKey = CScript() << OP_1; + hash = tx.GetHash(); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx.vout[0].scriptPubKey = CScript() << OP_2; + hash = tx.GetHash(); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); } - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); - // invalid p2sh txn in *m_node.mempool, template creation fails - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vin[0].prevout.n = 0; - tx.vin[0].scriptSig = CScript() << OP_1; - tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; - CScript script = CScript() << OP_0; - tx.vout[0].scriptPubKey = GetScriptForDestination(ScriptHash(script)); - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - tx.vin[0].prevout.hash = hash; - tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); - tx.vout[0].nValue -= LOWFEE; - hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - // Should throw block-validation-failed - BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); - m_node.mempool->clear(); - - // Delete the dummy blocks again. - while (m_node.chainman->ActiveChain().Tip()->nHeight > nHeight) { - CBlockIndex* del = m_node.chainman->ActiveChain().Tip(); - m_node.chainman->ActiveChain().SetTip(*Assert(del->pprev)); - m_node.chainman->ActiveChainstate().CoinsTip().SetBestBlock(del->pprev->GetBlockHash()); - delete del->phashBlock; - delete del; + { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + + // subsidy changing + int nHeight = m_node.chainman->ActiveChain().Height(); + // Create an actual 209999-long block chain (without valid blocks). + while (m_node.chainman->ActiveChain().Tip()->nHeight < 209999) { + CBlockIndex* prev = m_node.chainman->ActiveChain().Tip(); + CBlockIndex* next = new CBlockIndex(); + next->phashBlock = new uint256(InsecureRand256()); + m_node.chainman->ActiveChainstate().CoinsTip().SetBestBlock(next->GetBlockHash()); + next->pprev = prev; + next->nHeight = prev->nHeight + 1; + next->BuildSkip(); + m_node.chainman->ActiveChain().SetTip(*next); + } + BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + // Extend to a 210000-long block chain. + while (m_node.chainman->ActiveChain().Tip()->nHeight < 210000) { + CBlockIndex* prev = m_node.chainman->ActiveChain().Tip(); + CBlockIndex* next = new CBlockIndex(); + next->phashBlock = new uint256(InsecureRand256()); + m_node.chainman->ActiveChainstate().CoinsTip().SetBestBlock(next->GetBlockHash()); + next->pprev = prev; + next->nHeight = prev->nHeight + 1; + next->BuildSkip(); + m_node.chainman->ActiveChain().SetTip(*next); + } + BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); + + // invalid p2sh txn in tx_mempool, template creation fails + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vin[0].prevout.n = 0; + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vout[0].nValue = BLOCKSUBSIDY - LOWFEE; + CScript script = CScript() << OP_0; + tx.vout[0].scriptPubKey = GetScriptForDestination(ScriptHash(script)); + hash = tx.GetHash(); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); + tx.vout[0].nValue -= LOWFEE; + hash = tx.GetHash(); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + // Should throw block-validation-failed + BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); + + // Delete the dummy blocks again. + while (m_node.chainman->ActiveChain().Tip()->nHeight > nHeight) { + CBlockIndex* del = m_node.chainman->ActiveChain().Tip(); + m_node.chainman->ActiveChain().SetTip(*Assert(del->pprev)); + m_node.chainman->ActiveChainstate().CoinsTip().SetBestBlock(del->pprev->GetBlockHash()); + delete del->phashBlock; + delete del; + } } + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + // non-final txs in mempool SetMockTime(m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1); const int flags{LOCKTIME_VERIFY_SEQUENCE}; @@ -388,9 +432,9 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C tx.vout[0].scriptPubKey = CScript() << OP_1; tx.nLockTime = 0; hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes - BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail { CBlockIndex* active_chain_tip = m_node.chainman->ActiveChain().Tip(); @@ -402,9 +446,9 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | (((m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1-m_node.chainman->ActiveChain()[1]->GetMedianTimePast()) >> CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) + 1); // txFirst[1] is the 3rd block prevheights[0] = baseheight + 2; hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes - BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail const int SEQUENCE_LOCK_TIME = 512; // Sequence locks pass 512 seconds later for (int i = 0; i < CBlockIndex::nMedianTimeSpan; ++i) @@ -425,9 +469,9 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C prevheights[0] = baseheight + 3; tx.nLockTime = m_node.chainman->ActiveChain().Tip()->nHeight + 1; hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); BOOST_CHECK(!CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime fails - BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass + BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block // absolute time locked @@ -436,9 +480,9 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C prevheights.resize(1); prevheights[0] = baseheight + 4; hash = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); BOOST_CHECK(!CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime fails - BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass + BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later // mempool-dependent transactions (not added) @@ -447,15 +491,16 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C tx.nLockTime = 0; tx.vin[0].nSequence = 0; BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes - BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass + BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass tx.vin[0].nSequence = 1; - BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG; - BOOST_CHECK(TestSequenceLocks(CTransaction{tx})); // Sequence locks pass + BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | 1; - BOOST_CHECK(!TestSequenceLocks(CTransaction{tx})); // Sequence locks fail + BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); + BOOST_CHECK(pblocktemplate); // None of the of the absolute height/time locked tx should have made // it into the template because we still check IsFinalTx in CreateNewBlock, @@ -470,12 +515,15 @@ void MinerTestingSetup::TestBasicMining(const CChainParams& chainparams, const C m_node.chainman->ActiveChain().Tip()->nHeight++; SetMockTime(m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1); - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); BOOST_CHECK_EQUAL(pblocktemplate->block.vtx.size(), 5U); } -void MinerTestingSetup::TestPrioritisedMining(const CChainParams& chainparams, const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) +void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst) { + CTxMemPool& tx_mempool{MakeMempool()}; + LOCK(tx_mempool.cs); + TestMemPoolEntryHelper entry; // Test that a tx below min fee but prioritised is included @@ -487,29 +535,29 @@ void MinerTestingSetup::TestPrioritisedMining(const CChainParams& chainparams, c tx.vout.resize(1); tx.vout[0].nValue = 5000000000LL; // 0 fee uint256 hashFreePrioritisedTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(0).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - m_node.mempool->PrioritiseTransaction(hashFreePrioritisedTx, 5 * COIN); + tx_mempool.addUnchecked(entry.Fee(0).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.PrioritiseTransaction(hashFreePrioritisedTx, 5 * COIN); tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vin[0].prevout.n = 0; tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis uint256 hashParentTx = tx.GetHash(); // save this txid for later use - m_node.mempool->addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[2]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; uint256 hashMediumFeeTx = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - m_node.mempool->PrioritiseTransaction(hashMediumFeeTx, -5 * COIN); + tx_mempool.addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.PrioritiseTransaction(hashMediumFeeTx, -5 * COIN); // This tx also has a low fee, but is prioritised tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 1000; // 1000 satoshi fee uint256 hashPrioritsedChild = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - m_node.mempool->PrioritiseTransaction(hashPrioritsedChild, 2 * COIN); + tx_mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.PrioritiseTransaction(hashPrioritsedChild, 2 * COIN); // Test that transaction selection properly updates ancestor fee calculations as prioritised // parents get included in a block. Create a transaction with two prioritised ancestors, each @@ -520,21 +568,21 @@ void MinerTestingSetup::TestPrioritisedMining(const CChainParams& chainparams, c tx.vin[0].prevout.hash = txFirst[3]->GetHash(); tx.vout[0].nValue = 5000000000LL; // 0 fee uint256 hashFreeParent = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); - m_node.mempool->PrioritiseTransaction(hashFreeParent, 10 * COIN); + tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.PrioritiseTransaction(hashFreeParent, 10 * COIN); tx.vin[0].prevout.hash = hashFreeParent; tx.vout[0].nValue = 5000000000LL; // 0 fee uint256 hashFreeChild = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); - m_node.mempool->PrioritiseTransaction(hashFreeChild, 1 * COIN); + tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.PrioritiseTransaction(hashFreeChild, 1 * COIN); tx.vin[0].prevout.hash = hashFreeChild; tx.vout[0].nValue = 5000000000LL; // 0 fee uint256 hashFreeGrandchild = tx.GetHash(); - m_node.mempool->addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); - auto pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6U); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashFreeParent); BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashFreePrioritisedTx); @@ -553,15 +601,12 @@ void MinerTestingSetup::TestPrioritisedMining(const CChainParams& chainparams, c BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { // Note that by default, these tests run with size accounting enabled. - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); - const CChainParams& chainparams = *chainParams; CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; std::unique_ptr<CBlockTemplate> pblocktemplate; - fCheckpointsEnabled = false; - + CTxMemPool& tx_mempool{*m_node.mempool}; // Simple block creation, nothing special yet: - BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + BOOST_CHECK(pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); // We can't make transactions until we have inputs // Therefore, load 110 blocks :) @@ -593,23 +638,18 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) } LOCK(cs_main); - LOCK(m_node.mempool->cs); - TestBasicMining(chainparams, scriptPubKey, txFirst, baseheight); + TestBasicMining(scriptPubKey, txFirst, baseheight); m_node.chainman->ActiveChain().Tip()->nHeight--; SetMockTime(0); - m_node.mempool->clear(); - TestPackageSelection(chainparams, scriptPubKey, txFirst); + TestPackageSelection(scriptPubKey, txFirst); m_node.chainman->ActiveChain().Tip()->nHeight--; SetMockTime(0); - m_node.mempool->clear(); - - TestPrioritisedMining(chainparams, scriptPubKey, txFirst); - fCheckpointsEnabled = true; + TestPrioritisedMining(scriptPubKey, txFirst); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/minisketch_tests.cpp b/src/test/minisketch_tests.cpp index 9c53ace633..81f2aad623 100644 --- a/src/test/minisketch_tests.cpp +++ b/src/test/minisketch_tests.cpp @@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(minisketch_test) Minisketch sketch_c = std::move(sketch_ar); sketch_c.Merge(sketch_br); auto dec = sketch_c.Decode(errors); - BOOST_CHECK(dec.has_value()); + BOOST_REQUIRE(dec.has_value()); auto sols = std::move(*dec); std::sort(sols.begin(), sols.end()); for (uint32_t i = 0; i < a_not_b; ++i) BOOST_CHECK_EQUAL(sols[i], start_a + i); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index c2d2fa37b4..0e1e9ae211 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -84,12 +84,12 @@ BOOST_AUTO_TEST_CASE(netbase_properties) } -bool static TestSplitHost(const std::string& test, const std::string& host, uint16_t port) +bool static TestSplitHost(const std::string& test, const std::string& host, uint16_t port, bool validPort=true) { std::string hostOut; uint16_t portOut{0}; - SplitHostPort(test, portOut, hostOut); - return hostOut == host && port == portOut; + bool validPortOut = SplitHostPort(test, portOut, hostOut); + return hostOut == host && portOut == port && validPortOut == validPort; } BOOST_AUTO_TEST_CASE(netbase_splithost) @@ -109,6 +109,23 @@ BOOST_AUTO_TEST_CASE(netbase_splithost) BOOST_CHECK(TestSplitHost(":8333", "", 8333)); BOOST_CHECK(TestSplitHost("[]:8333", "", 8333)); BOOST_CHECK(TestSplitHost("", "", 0)); + BOOST_CHECK(TestSplitHost(":65535", "", 65535)); + BOOST_CHECK(TestSplitHost(":65536", ":65536", 0, false)); + BOOST_CHECK(TestSplitHost(":-1", ":-1", 0, false)); + BOOST_CHECK(TestSplitHost("[]:70001", "[]:70001", 0, false)); + BOOST_CHECK(TestSplitHost("[]:-1", "[]:-1", 0, false)); + BOOST_CHECK(TestSplitHost("[]:-0", "[]:-0", 0, false)); + BOOST_CHECK(TestSplitHost("[]:0", "", 0, false)); + BOOST_CHECK(TestSplitHost("[]:1/2", "[]:1/2", 0, false)); + BOOST_CHECK(TestSplitHost("[]:1E2", "[]:1E2", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:65536", "127.0.0.1:65536", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:0", "127.0.0.1", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:", "127.0.0.1:", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:1/2", "127.0.0.1:1/2", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:1E2", "127.0.0.1:1E2", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:65536", "www.bitcoincore.org:65536", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:0", "www.bitcoincore.org", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:", "www.bitcoincore.org:", 0, false)); } bool static TestParse(std::string src, std::string canon) diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 3f66a8fc46..b652dc44c0 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -4,6 +4,7 @@ #include <policy/fees.h> #include <policy/policy.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <uint256.h> #include <util/time.h> diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index c88cd36688..d362c85560 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/rbf.h> #include <random.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/system.h> #include <util/time.h> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 55486db6b2..d78f62972f 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <coins.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <key.h> diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index f160bb08a5..11f4be7fef 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <test/util/setup_common.h> -#include <util/system.h> +#include <common/run_command.h> #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 62c7ddb673..643d9221fe 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -69,11 +69,16 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) } } + // It is not safe to stop and destroy the index until it finishes handling + // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain() + // call above is sufficient to ensure this, but the + // SyncWithValidationInterfaceQueue() call below is also needed to ensure + // TSAN always sees the test thread waiting for the notification thread, and + // avoid potential false positive reports. + SyncWithValidationInterfaceQueue(); + // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - - // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed - SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txreconciliation_tests.cpp b/src/test/txreconciliation_tests.cpp new file mode 100644 index 0000000000..bd74998002 --- /dev/null +++ b/src/test/txreconciliation_tests.cpp @@ -0,0 +1,86 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txreconciliation.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(txreconciliation_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(RegisterPeerTest) +{ + TxReconciliationTracker tracker(1); + const uint64_t salt = 0; + + // Prepare a peer for reconciliation. + tracker.PreRegisterPeer(0); + + // Both roles are false, don't register. + BOOST_CHECK(tracker.RegisterPeer(/*peer_id=*/0, /*is_peer_inbound=*/true, + /*is_peer_recon_initiator=*/false, + /*is_peer_recon_responder=*/false, + /*peer_recon_version=*/1, salt) == + ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Invalid roles for the given connection direction. + BOOST_CHECK(tracker.RegisterPeer(0, true, false, true, 1, salt) == ReconciliationRegisterResult::PROTOCOL_VIOLATION); + BOOST_CHECK(tracker.RegisterPeer(0, false, true, false, 1, salt) == ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Invalid version. + BOOST_CHECK(tracker.RegisterPeer(0, true, true, false, 0, salt) == ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Valid registration. + BOOST_REQUIRE(!tracker.IsPeerRegistered(0)); + BOOST_REQUIRE(tracker.RegisterPeer(0, true, true, false, 1, salt) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(0)); + + // Reconciliation version is higher than ours, should be able to register. + BOOST_REQUIRE(!tracker.IsPeerRegistered(1)); + tracker.PreRegisterPeer(1); + BOOST_REQUIRE(tracker.RegisterPeer(1, true, true, false, 2, salt) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(1)); + + // Do not register if there were no pre-registration for the peer. + BOOST_REQUIRE(tracker.RegisterPeer(100, true, true, false, 1, salt) == ReconciliationRegisterResult::NOT_FOUND); + BOOST_CHECK(!tracker.IsPeerRegistered(100)); +} + +BOOST_AUTO_TEST_CASE(ForgetPeerTest) +{ + TxReconciliationTracker tracker(1); + NodeId peer_id0 = 0; + + // Removing peer after pre-registring works and does not let to register the peer. + tracker.PreRegisterPeer(peer_id0); + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(tracker.RegisterPeer(peer_id0, true, true, false, 1, 1) == ReconciliationRegisterResult::NOT_FOUND); + + // Removing peer after it is registered works. + tracker.PreRegisterPeer(peer_id0); + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + BOOST_REQUIRE(tracker.RegisterPeer(peer_id0, true, true, false, 1, 1) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(peer_id0)); + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(!tracker.IsPeerRegistered(peer_id0)); +} + +BOOST_AUTO_TEST_CASE(IsPeerRegisteredTest) +{ + TxReconciliationTracker tracker(1); + NodeId peer_id0 = 0; + + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + tracker.PreRegisterPeer(peer_id0); + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + + BOOST_REQUIRE(tracker.RegisterPeer(peer_id0, true, true, false, 1, 1) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(peer_id0)); + + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(!tracker.IsPeerRegistered(peer_id0)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 2f0021b114..0ca63810f3 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -11,6 +11,7 @@ #include <node/context.h> #include <node/utxo_snapshot.h> #include <rpc/blockchain.h> +#include <test/util/setup_common.h> #include <validation.h> #include <univalue.h> @@ -20,11 +21,24 @@ const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to * malleate the snapshot. + * + * If `reset_chainstate` is true, reset the original chainstate back to the genesis + * block. This allows us to simulate more realistic conditions in which a snapshot is + * loaded into an otherwise mostly-uninitialized datadir. It also allows us to test + * conditions that would otherwise cause shutdowns based on the IBD chainstate going + * past the snapshot it generated. */ template<typename F = decltype(NoMalleation)> static bool -CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F malleation = NoMalleation) +CreateAndActivateUTXOSnapshot( + TestingSetup* fixture, + F malleation = NoMalleation, + bool reset_chainstate = false, + bool in_memory_chainstate = false) { + node::NodeContext& node = fixture->m_node; + fs::path root = fixture->m_path_root; + // Write out a snapshot to the test's tempdir. // int height; @@ -47,7 +61,38 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma malleation(auto_infile, metadata); - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory=*/true); + if (reset_chainstate) { + { + // What follows is code to selectively reset chainstate data without + // disturbing the existing BlockManager instance, which is needed to + // recognize the headers chain previously generated by the chainstate we're + // removing. Without those headers, we can't activate the snapshot below. + // + // This is a stripped-down version of node::LoadChainstate which + // preserves the block index. + LOCK(::cs_main); + uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash(); + node.chainman->ResetChainstates(); + node.chainman->InitializeChainstate(node.mempool.get()); + Chainstate& chain = node.chainman->ActiveChainstate(); + Assert(chain.LoadGenesisBlock()); + // These cache values will be corrected shortly in `MaybeRebalanceCaches`. + chain.InitCoinsDB(1 << 20, true, false, ""); + chain.InitCoinsCache(1 << 20); + chain.CoinsTip().SetBestBlock(gen_hash); + chain.setBlockIndexCandidates.insert(node.chainman->m_blockman.LookupBlockIndex(gen_hash)); + chain.LoadChainTip(); + node.chainman->MaybeRebalanceCaches(); + } + BlockValidationState state; + if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); + } + Assert( + 0 == WITH_LOCK(node.chainman->GetMutex(), return node.chainman->ActiveHeight())); + } + + return node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate); } diff --git a/src/test/util/net.h b/src/test/util/net.h index 73543de4ca..9ae7981980 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -166,6 +166,10 @@ public: return 0; } + bool SetNonBlocking() const override { return true; } + + bool IsSelectable() const override { return true; } + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 74b055ee45..9f29342b10 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -36,6 +36,7 @@ #include <shutdown.h> #include <streams.h> #include <test/util/net.h> +#include <test/util/txmempool.h> #include <timedata.h> #include <txdb.h> #include <txmempool.h> @@ -60,7 +61,6 @@ using node::ApplyArgsManOptions; using node::BlockAssembler; using node::CalculateCacheSizes; using node::LoadChainstate; -using node::NodeContext; using node::RegenerateCommitments; using node::VerifyLoadedChainstate; @@ -162,19 +162,6 @@ BasicTestingSetup::~BasicTestingSetup() gArgs.ClearArgs(); } -CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) -{ - CTxMemPool::Options mempool_opts{ - .estimator = node.fee_estimator.get(), - // Default to always checking mempool regardless of - // chainparams.DefaultConsistencyChecks for tests - .check_ratio = 1, - }; - const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; - Assert(!err); - return mempool_opts; -} - ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : BasicTestingSetup(chainName, extra_args) { @@ -220,17 +207,12 @@ ChainTestingSetup::~ChainTestingSetup() m_node.chainman.reset(); } -TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) - : ChainTestingSetup(chainName, extra_args) +void TestingSetup::LoadVerifyActivateChainstate() { - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - RegisterAllCoreRPCCommands(tableRPC); - node::ChainstateLoadOptions options; options.mempool = Assert(m_node.mempool.get()); - options.block_tree_db_in_memory = true; - options.coins_db_in_memory = true; + options.block_tree_db_in_memory = m_block_tree_db_in_memory; + options.coins_db_in_memory = m_coins_db_in_memory; options.reindex = node::fReindex; options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); options.prune = node::fPruneMode; @@ -246,6 +228,22 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } +} + +TestingSetup::TestingSetup( + const std::string& chainName, + const std::vector<const char*>& extra_args, + const bool coins_db_in_memory, + const bool block_tree_db_in_memory) + : ChainTestingSetup(chainName, extra_args), + m_coins_db_in_memory(coins_db_in_memory), + m_block_tree_db_in_memory(block_tree_db_in_memory) +{ + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + RegisterAllCoreRPCCommands(tableRPC); + + LoadVerifyActivateChainstate(); m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>()); m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman, @@ -263,8 +261,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup(const std::string& chain_name, const std::vector<const char*>& extra_args) - : TestingSetup{chain_name, extra_args} +TestChain100Setup::TestChain100Setup( + const std::string& chain_name, + const std::vector<const char*>& extra_args, + const bool coins_db_in_memory, + const bool block_tree_db_in_memory) + : TestingSetup{CBaseChainParams::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { @@ -423,17 +425,6 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex return mempool_transactions; } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const -{ - return FromTx(MakeTransactionRef(tx)); -} - -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const -{ - return CTxMemPoolEntry(tx, nFee, nTime, nHeight, - spendsCoinbase, sigOpCost, lp); -} - /** * @returns a real block (0000000000013b8ab2cd513b0261a14096412195a72a0c4827d229dcc7e0f7af) * with 9 txs. diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 136ee1fd62..dfa36039a2 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -8,21 +8,23 @@ #include <chainparamsbase.h> #include <fs.h> #include <key.h> -#include <util/system.h> #include <node/caches.h> #include <node/context.h> +#include <primitives/transaction.h> #include <pubkey.h> #include <random.h> #include <stdexcept> -#include <txmempool.h> #include <util/check.h> #include <util/string.h> +#include <util/system.h> #include <util/vector.h> #include <functional> #include <type_traits> #include <vector> +class Chainstate; + /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; @@ -90,9 +92,6 @@ struct BasicTestingSetup { ArgsManager m_args; }; - -CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); - /** Testing setup that performs all steps up until right before * ChainstateManager gets initialized. Meant for testing ChainstateManager * initialization behaviour. @@ -107,7 +106,16 @@ struct ChainTestingSetup : public BasicTestingSetup { /** Testing setup that configures a complete environment. */ struct TestingSetup : public ChainTestingSetup { - explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + bool m_coins_db_in_memory{true}; + bool m_block_tree_db_in_memory{true}; + + void LoadVerifyActivateChainstate(); + + explicit TestingSetup( + const std::string& chainName = CBaseChainParams::MAIN, + const std::vector<const char*>& extra_args = {}, + const bool coins_db_in_memory = true, + const bool block_tree_db_in_memory = true); }; /** Identical to TestingSetup, but chain set to regtest */ @@ -124,8 +132,11 @@ class CScript; * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ struct TestChain100Setup : public TestingSetup { - TestChain100Setup(const std::string& chain_name = CBaseChainParams::REGTEST, - const std::vector<const char*>& extra_args = {}); + TestChain100Setup( + const std::string& chain_name = CBaseChainParams::REGTEST, + const std::vector<const char*>& extra_args = {}, + const bool coins_db_in_memory = true, + const bool block_tree_db_in_memory = true); /** * Create a new block with just given transactions, coinbase paying to @@ -201,33 +212,6 @@ std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseCha return std::make_unique<T>(chain_name, arguments); } -class CTxMemPoolEntry; - -struct TestMemPoolEntryHelper -{ - // Default values - CAmount nFee; - int64_t nTime; - unsigned int nHeight; - bool spendsCoinbase; - unsigned int sigOpCost; - LockPoints lp; - - TestMemPoolEntryHelper() : - nFee(0), nTime(0), nHeight(1), - spendsCoinbase(false), sigOpCost(4) { } - - CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; - CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; - - // Change the default value - TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } - TestMemPoolEntryHelper &Time(int64_t _time) { nTime = _time; return *this; } - TestMemPoolEntryHelper &Height(unsigned int _height) { nHeight = _height; return *this; } - TestMemPoolEntryHelper &SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } - TestMemPoolEntryHelper &SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } -}; - CBlock getBlock13b8a(); // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_* diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp new file mode 100644 index 0000000000..12cc1a4a3d --- /dev/null +++ b/src/test/util/txmempool.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/txmempool.h> + +#include <chainparams.h> +#include <node/context.h> +#include <node/mempool_args.h> +#include <txmempool.h> +#include <util/check.h> +#include <util/time.h> +#include <util/translation.h> + +using node::NodeContext; + +CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) +{ + CTxMemPool::Options mempool_opts{ + .estimator = node.fee_estimator.get(), + // Default to always checking mempool regardless of + // chainparams.DefaultConsistencyChecks for tests + .check_ratio = 1, + }; + const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; + Assert(!err); + return mempool_opts; +} + +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const +{ + return FromTx(MakeTransactionRef(tx)); +} + +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const +{ + return CTxMemPoolEntry(tx, nFee, nTime, nHeight, + spendsCoinbase, sigOpCost, lp); +} diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h new file mode 100644 index 0000000000..70b9ed88db --- /dev/null +++ b/src/test/util/txmempool.h @@ -0,0 +1,37 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_TXMEMPOOL_H +#define BITCOIN_TEST_UTIL_TXMEMPOOL_H + +#include <txmempool.h> + +namespace node { +struct NodeContext; +} + +CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); + +struct TestMemPoolEntryHelper +{ + // Default values + CAmount nFee{0}; + int64_t nTime{0}; + unsigned int nHeight{1}; + bool spendsCoinbase{false}; + unsigned int sigOpCost{4}; + LockPoints lp; + + CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; + CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; + + // Change the default value + TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } + TestMemPoolEntryHelper &Time(int64_t _time) { nTime = _time; return *this; } + TestMemPoolEntryHelper &Height(unsigned int _height) { nHeight = _height; return *this; } + TestMemPoolEntryHelper &SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } + TestMemPoolEntryHelper &SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } +}; + +#endif // BITCOIN_TEST_UTIL_TXMEMPOOL_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 6e59d2e8e6..009c27927f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -26,6 +26,7 @@ #include <util/bitdeque.h> #include <array> +#include <cmath> #include <fstream> #include <limits> #include <map> diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 347a967b33..f868c0d4e6 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -89,7 +89,8 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // After adding some blocks to the tip, best block should have changed. BOOST_CHECK(::g_best_block != curr_tip); - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot( + this, NoMalleation, /*reset_chainstate=*/ true)); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 9fcb7d315a..22b9af1201 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -10,6 +10,7 @@ #include <sync.h> #include <test/util/chainstate.h> #include <test/util/setup_common.h> +#include <timedata.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> @@ -63,7 +64,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // const uint256 snapshot_blockhash = GetRandHash(); - Chainstate& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( + Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot( &mempool, snapshot_blockhash)); chainstates.push_back(&c2); @@ -133,7 +134,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - Chainstate& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); + Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -154,162 +155,240 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -//! Test basic snapshot activation. -BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) -{ - ChainstateManager& chainman = *Assert(m_node.chainman); - - size_t initial_size; - size_t initial_total_coins{100}; - - // Make some initial assertions about the contents of the chainstate. +struct SnapshotTestSetup : TestChain100Setup { + // Run with coinsdb on the filesystem to support, e.g., moving invalidated + // chainstate dirs to "*_invalid". + // + // Note that this means the tests run considerably slower than in-memory DB + // tests, but we can't otherwise test this functionality since it relies on + // destructive filesystem operations. + SnapshotTestSetup() : TestChain100Setup{ + {}, + {}, + /*coins_db_in_memory=*/false, + /*block_tree_db_in_memory=*/false, + } { - LOCK(::cs_main); - CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); - initial_size = ibd_coinscache.GetCacheSize(); - size_t total_coins{0}; - - for (CTransactionRef& txn : m_coinbase_txns) { - COutPoint op{txn->GetHash(), 0}; - BOOST_CHECK(ibd_coinscache.HaveCoin(op)); - total_coins++; - } - - BOOST_CHECK_EQUAL(total_coins, initial_total_coins); - BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } - // Snapshot should refuse to load at this height. - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); - BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); - BOOST_CHECK(!chainman.SnapshotBlockhash()); - - // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can - // be found. - constexpr int snapshot_height = 110; - mineBlocks(10); - initial_size += 10; - initial_total_coins += 10; - - // Should not load malleated snapshots - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // A UTXO is missing but count is correct - metadata.m_coins_count -= 1; - - COutPoint outpoint; - Coin coin; - - auto_infile >> outpoint; - auto_infile >> coin; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Coins count is larger than coins in file - metadata.m_coins_count += 1; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Coins count is smaller than coins in file - metadata.m_coins_count -= 1; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Wrong hash - metadata.m_base_blockhash = uint256::ZERO; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Wrong hash - metadata.m_base_blockhash = uint256::ONE; - })); - - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); - - // Ensure our active chain is the snapshot chainstate. - BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); - BOOST_CHECK_EQUAL( - *chainman.ActiveChainstate().m_from_snapshot_blockhash, - *chainman.SnapshotBlockhash()); - - // Ensure that the genesis block was not marked assumed-valid. - BOOST_CHECK(WITH_LOCK(::cs_main, return !chainman.ActiveChain().Genesis()->IsAssumedValid())); - - const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); - const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); - - BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); - - // To be checked against later when we try loading a subsequent snapshot. - uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; - - // Make some assertions about the both chainstates. These checks ensure the - // legacy chainstate hasn't changed and that the newly created chainstate - // reflects the expected content. + std::tuple<Chainstate*, Chainstate*> SetupSnapshot() { - LOCK(::cs_main); - int chains_tested{0}; + ChainstateManager& chainman = *Assert(m_node.chainman); - for (Chainstate* chainstate : chainman.GetAll()) { - BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); - CCoinsViewCache& coinscache = chainstate->CoinsTip(); + BOOST_CHECK(!chainman.IsSnapshotActive()); - // Both caches will be empty initially. - BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + { + LOCK(::cs_main); + BOOST_CHECK(!chainman.IsSnapshotValidated()); + BOOST_CHECK(!node::FindSnapshotChainstateDir()); + } + size_t initial_size; + size_t initial_total_coins{100}; + + // Make some initial assertions about the contents of the chainstate. + { + LOCK(::cs_main); + CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); + initial_size = ibd_coinscache.GetCacheSize(); size_t total_coins{0}; for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; - BOOST_CHECK(coinscache.HaveCoin(op)); + BOOST_CHECK(ibd_coinscache.HaveCoin(op)); total_coins++; } - BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); BOOST_CHECK_EQUAL(total_coins, initial_total_coins); - chains_tested++; + BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } - BOOST_CHECK_EQUAL(chains_tested, 2); - } + Chainstate& validation_chainstate = chainman.ActiveChainstate(); + + // Snapshot should refuse to load at this height. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); + BOOST_CHECK(!chainman.SnapshotBlockhash()); + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + constexpr int snapshot_height = 110; + mineBlocks(10); + initial_size += 10; + initial_total_coins += 10; + + // Should not load malleated snapshots + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // A UTXO is missing but count is correct + metadata.m_coins_count -= 1; + + COutPoint outpoint; + Coin coin; + + auto_infile >> outpoint; + auto_infile >> coin; + })); + + BOOST_CHECK(!node::FindSnapshotChainstateDir()); + + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is larger than coins in file + metadata.m_coins_count += 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is smaller than coins in file + metadata.m_coins_count -= 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Wrong hash + metadata.m_base_blockhash = uint256::ZERO; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Wrong hash + metadata.m_base_blockhash = uint256::ONE; + })); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); + BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); + BOOST_CHECK_EQUAL( + *chainman.ActiveChainstate().m_from_snapshot_blockhash, + *chainman.SnapshotBlockhash()); + + Chainstate& snapshot_chainstate = chainman.ActiveChainstate(); + + { + LOCK(::cs_main); - // Mine some new blocks on top of the activated snapshot chainstate. - constexpr size_t new_coins{100}; - mineBlocks(new_coins); // Defined in TestChain100Setup. + fs::path found = *node::FindSnapshotChainstateDir(); - { - LOCK(::cs_main); - size_t coins_in_active{0}; - size_t coins_in_background{0}; - size_t coins_missing_from_background{0}; + // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. + BOOST_CHECK_EQUAL( + *node::ReadSnapshotBaseBlockhash(found), + *chainman.SnapshotBlockhash()); - for (Chainstate* chainstate : chainman.GetAll()) { - BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); - CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_background = chainstate != &chainman.ActiveChainstate(); + // Ensure that the genesis block was not marked assumed-valid. + BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); + } - for (CTransactionRef& txn : m_coinbase_txns) { - COutPoint op{txn->GetHash(), 0}; - if (coinscache.HaveCoin(op)) { - (is_background ? coins_in_background : coins_in_active)++; - } else if (is_background) { - coins_missing_from_background++; + const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); + const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); + + BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); + + // To be checked against later when we try loading a subsequent snapshot. + uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; + + // Make some assertions about the both chainstates. These checks ensure the + // legacy chainstate hasn't changed and that the newly created chainstate + // reflects the expected content. + { + LOCK(::cs_main); + int chains_tested{0}; + + for (Chainstate* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + + // Both caches will be empty initially. + BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(coinscache.HaveCoin(op)); + total_coins++; } + + BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + chains_tested++; } + + BOOST_CHECK_EQUAL(chains_tested, 2); } - BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); + // Mine some new blocks on top of the activated snapshot chainstate. + constexpr size_t new_coins{100}; + mineBlocks(new_coins); // Defined in TestChain100Setup. + + { + LOCK(::cs_main); + size_t coins_in_active{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; + + for (Chainstate* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + bool is_background = chainstate != &chainman.ActiveChainstate(); + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + if (coinscache.HaveCoin(op)) { + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; + } + } + } + + BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); + } + + // Snapshot should refuse to load after one has already loaded. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); + + // Snapshot blockhash should be unchanged. + BOOST_CHECK_EQUAL( + *chainman.ActiveChainstate().m_from_snapshot_blockhash, + loaded_snapshot_blockhash); + return std::make_tuple(&validation_chainstate, &snapshot_chainstate); } - // Snapshot should refuse to load after one has already loaded. - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + // Simulate a restart of the node by flushing all state to disk, clearing the + // existing ChainstateManager, and unloading the block index. + // + // @returns a reference to the "restarted" ChainstateManager + ChainstateManager& SimulateNodeRestart() + { + ChainstateManager& chainman = *Assert(m_node.chainman); + + BOOST_TEST_MESSAGE("Simulating node restart"); + { + LOCK(::cs_main); + for (Chainstate* cs : chainman.GetAll()) { + cs->ForceFlushStateToDisk(); + } + chainman.ResetChainstates(); + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); + const ChainstateManager::Options chainman_opts{ + .chainparams = ::Params(), + .adjusted_time_callback = GetAdjustedTime, + }; + // For robustness, ensure the old manager is destroyed before creating a + // new one. + m_node.chainman.reset(); + m_node.chainman.reset(new ChainstateManager(chainman_opts)); + } + return *Assert(m_node.chainman); + } +}; - // Snapshot blockhash should be unchanged. - BOOST_CHECK_EQUAL( - *chainman.ActiveChainstate().m_from_snapshot_blockhash, - loaded_snapshot_blockhash); +//! Test basic snapshot activation. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) +{ + this->SetupSnapshot(); } //! Test LoadBlockIndex behavior when multiple chainstates are in use. @@ -374,7 +453,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); Chainstate& cs2 = WITH_LOCK(::cs_main, - return chainman.InitializeChainstate(&mempool, GetRandHash())); + return chainman.ActivateExistingSnapshot(&mempool, GetRandHash())); reload_all_block_indexes(); @@ -390,4 +469,59 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); } +//! Ensure that snapshot chainstates initialize properly when found on disk. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) +{ + this->SetupSnapshot(); + + ChainstateManager& chainman = *Assert(m_node.chainman); + + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); + BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); + + BOOST_CHECK(chainman.IsSnapshotActive()); + const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(), + return chainman.ActiveTip()->GetBlockHash()); + + auto all_chainstates = chainman.GetAll(); + BOOST_CHECK_EQUAL(all_chainstates.size(), 2); + + // Test that simulating a shutdown (resetting ChainstateManager) and then performing + // chainstate reinitializing successfully cleans up the background-validation + // chainstate data, and we end up with a single chainstate that is at tip. + ChainstateManager& chainman_restarted = this->SimulateNodeRestart(); + + BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); + + // This call reinitializes the chainstates. + this->LoadVerifyActivateChainstate(); + + { + LOCK(chainman_restarted.GetMutex()); + BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 2); + BOOST_CHECK(chainman_restarted.IsSnapshotActive()); + BOOST_CHECK(!chainman_restarted.IsSnapshotValidated()); + + BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); + BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); + } + + BOOST_TEST_MESSAGE( + "Ensure we can mine blocks on top of the initialized snapshot chainstate"); + mineBlocks(10); + { + LOCK(chainman_restarted.GetMutex()); + BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); + + // Background chainstate should be unaware of new blocks on the snapshot + // chainstate. + for (Chainstate* cs : chainman_restarted.GetAll()) { + if (cs != &chainman_restarted.ActiveChainstate()) { + BOOST_CHECK_EQUAL(cs->m_chain.Height(), 110); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.h b/src/txdb.h index a04596f7bb..8c41e26f6a 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -9,6 +9,7 @@ #include <coins.h> #include <dbwrapper.h> #include <sync.h> +#include <fs.h> #include <memory> #include <optional> @@ -72,6 +73,9 @@ public: //! Dynamically alter the underlying leveldb cache size. void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! @returns filesystem path to on-disk storage or std::nullopt if in memory. + std::optional<fs::path> StoragePath() { return m_db->StoragePath(); } }; /** Access to the block database (blocks/index/) */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index e1288b7346..84ed2e9ef5 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -145,7 +145,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256>& vHashes // Iterate in reverse, so that whenever we are looking at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that - // CTxMemPool::m_children will be updated, an assumption made in + // CTxMemPoolEntry::m_children will be updated, an assumption made in // UpdateForDescendants. for (const uint256 &hash : reverse_iterate(vHashesToUpdate)) { // calculate children from mapNextTx @@ -154,7 +154,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256>& vHashes continue; } auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); - // First calculate the children, and update CTxMemPool::m_children to + // First calculate the children, and update CTxMemPoolEntry::m_children to // include them, and update their CTxMemPoolEntry::m_parents to include this tx. // we cache the in-mempool children to avoid duplicate updates { @@ -187,10 +187,7 @@ bool CTxMemPool::CalculateAncestorsAndCheckLimits(size_t entry_size, size_t entry_count, setEntries& setAncestors, CTxMemPoolEntry::Parents& staged_ancestors, - uint64_t limitAncestorCount, - uint64_t limitAncestorSize, - uint64_t limitDescendantCount, - uint64_t limitDescendantSize, + const Limits& limits, std::string &errString) const { size_t totalSizeWithAncestors = entry_size; @@ -203,14 +200,14 @@ bool CTxMemPool::CalculateAncestorsAndCheckLimits(size_t entry_size, staged_ancestors.erase(stage); totalSizeWithAncestors += stageit->GetTxSize(); - if (stageit->GetSizeWithDescendants() + entry_size > limitDescendantSize) { - errString = strprintf("exceeds descendant size limit for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limitDescendantSize); + if (stageit->GetSizeWithDescendants() + entry_size > static_cast<uint64_t>(limits.descendant_size_vbytes)) { + errString = strprintf("exceeds descendant size limit for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limits.descendant_size_vbytes); return false; - } else if (stageit->GetCountWithDescendants() + entry_count > limitDescendantCount) { - errString = strprintf("too many descendants for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limitDescendantCount); + } else if (stageit->GetCountWithDescendants() + entry_count > static_cast<uint64_t>(limits.descendant_count)) { + errString = strprintf("too many descendants for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limits.descendant_count); return false; - } else if (totalSizeWithAncestors > limitAncestorSize) { - errString = strprintf("exceeds ancestor size limit [limit: %u]", limitAncestorSize); + } else if (totalSizeWithAncestors > static_cast<uint64_t>(limits.ancestor_size_vbytes)) { + errString = strprintf("exceeds ancestor size limit [limit: %u]", limits.ancestor_size_vbytes); return false; } @@ -222,8 +219,8 @@ bool CTxMemPool::CalculateAncestorsAndCheckLimits(size_t entry_size, if (setAncestors.count(parent_it) == 0) { staged_ancestors.insert(parent); } - if (staged_ancestors.size() + setAncestors.size() + entry_count > limitAncestorCount) { - errString = strprintf("too many unconfirmed ancestors [limit: %u]", limitAncestorCount); + if (staged_ancestors.size() + setAncestors.size() + entry_count > static_cast<uint64_t>(limits.ancestor_count)) { + errString = strprintf("too many unconfirmed ancestors [limit: %u]", limits.ancestor_count); return false; } } @@ -233,10 +230,7 @@ bool CTxMemPool::CalculateAncestorsAndCheckLimits(size_t entry_size, } bool CTxMemPool::CheckPackageLimits(const Package& package, - uint64_t limitAncestorCount, - uint64_t limitAncestorSize, - uint64_t limitDescendantCount, - uint64_t limitDescendantSize, + const Limits& limits, std::string &errString) const { CTxMemPoolEntry::Parents staged_ancestors; @@ -247,8 +241,8 @@ bool CTxMemPool::CheckPackageLimits(const Package& package, std::optional<txiter> piter = GetIter(input.prevout.hash); if (piter) { staged_ancestors.insert(**piter); - if (staged_ancestors.size() + package.size() > limitAncestorCount) { - errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); + if (staged_ancestors.size() + package.size() > static_cast<uint64_t>(limits.ancestor_count)) { + errString = strprintf("too many unconfirmed parents [limit: %u]", limits.ancestor_count); return false; } } @@ -260,8 +254,7 @@ bool CTxMemPool::CheckPackageLimits(const Package& package, setEntries setAncestors; const auto ret = CalculateAncestorsAndCheckLimits(total_size, package.size(), setAncestors, staged_ancestors, - limitAncestorCount, limitAncestorSize, - limitDescendantCount, limitDescendantSize, errString); + limits, errString); // It's possible to overestimate the ancestor/descendant totals. if (!ret) errString.insert(0, "possibly "); return ret; @@ -269,10 +262,7 @@ bool CTxMemPool::CheckPackageLimits(const Package& package, bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, - uint64_t limitAncestorCount, - uint64_t limitAncestorSize, - uint64_t limitDescendantCount, - uint64_t limitDescendantSize, + const Limits& limits, std::string &errString, bool fSearchForParents /* = true */) const { @@ -287,8 +277,8 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, std::optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); if (piter) { staged_ancestors.insert(**piter); - if (staged_ancestors.size() + 1 > limitAncestorCount) { - errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); + if (staged_ancestors.size() + 1 > static_cast<uint64_t>(limits.ancestor_count)) { + errString = strprintf("too many unconfirmed parents [limit: %u]", limits.ancestor_count); return false; } } @@ -302,8 +292,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, return CalculateAncestorsAndCheckLimits(entry.GetTxSize(), /*entry_count=*/1, setAncestors, staged_ancestors, - limitAncestorCount, limitAncestorSize, - limitDescendantCount, limitDescendantSize, errString); + limits, errString); } void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) @@ -347,7 +336,6 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b { // For each entry, walk back all ancestors and decrement size associated with this // transaction - const uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); if (updateDescendants) { // updateDescendants should be true whenever we're not recursively // removing a tx and all its descendants, eg when a transaction is @@ -390,7 +378,7 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b // mempool parents we'd calculate by searching, and it's important that // we use the cached notion of ancestor transactions as the set of // things to update for removal. - CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); + CalculateMemPoolAncestors(entry, setAncestors, Limits::NoLimits(), dummy, false); // Note that UpdateAncestorsOf severs the child links that point to // removeIt in the entries for the parents of removeIt. UpdateAncestorsOf(false, removeIt, setAncestors); @@ -744,9 +732,8 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei assert(std::equal(setParentCheck.begin(), setParentCheck.end(), it->GetMemPoolParentsConst().begin(), comp)); // Verify ancestor state is correct. setEntries setAncestors; - uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); + CalculateMemPoolAncestors(*it, setAncestors, Limits::NoLimits(), dummy); uint64_t nCountCheck = setAncestors.size() + 1; uint64_t nSizeCheck = it->GetTxSize(); CAmount nFeesCheck = it->GetModifiedFee(); @@ -908,9 +895,8 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD mapTx.modify(it, [&nFeeDelta](CTxMemPoolEntry& e) { e.UpdateModifiedFee(nFeeDelta); }); // Now update all ancestors' modified fees with descendants setEntries setAncestors; - uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); + CalculateMemPoolAncestors(*it, setAncestors, Limits::NoLimits(), dummy, false); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, [=](CTxMemPoolEntry& e){ e.UpdateDescendantState(0, nFeeDelta, 0);}); } @@ -1049,9 +1035,8 @@ int CTxMemPool::Expire(std::chrono::seconds time) void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, bool validFeeEstimate) { setEntries setAncestors; - uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; - CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); + CalculateMemPoolAncestors(entry, setAncestors, Limits::NoLimits(), dummy); return addUnchecked(entry, setAncestors, validFeeEstimate); } diff --git a/src/txmempool.h b/src/txmempool.h index cd15d069b1..4afaac0506 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -526,6 +526,8 @@ public: typedef std::set<txiter, CompareIteratorByHash> setEntries; + using Limits = kernel::MemPoolLimits; + uint64_t CalculateDescendantMaximum(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); private: typedef std::map<txiter, setEntries, CompareIteratorByHash> cacheMap; @@ -545,19 +547,22 @@ private: /** * Helper function to calculate all in-mempool ancestors of staged_ancestors and apply ancestor * and descendant limits (including staged_ancestors thsemselves, entry_size and entry_count). - * param@[in] entry_size Virtual size to include in the limits. - * param@[in] entry_count How many entries to include in the limits. - * param@[in] staged_ancestors Should contain entries in the mempool. - * param@[out] setAncestors Will be populated with all mempool ancestors. + * + * @param[in] entry_size Virtual size to include in the limits. + * @param[in] entry_count How many entries to include in the limits. + * @param[out] setAncestors Will be populated with all mempool ancestors. + * @param[in] staged_ancestors Should contain entries in the mempool. + * @param[in] limits Maximum number and size of ancestors and descendants + * @param[out] errString Populated with error reason if any limits are hit + * + * @return true if no limits were hit and all in-mempool ancestors were calculated, false + * otherwise */ bool CalculateAncestorsAndCheckLimits(size_t entry_size, size_t entry_count, setEntries& setAncestors, CTxMemPoolEntry::Parents &staged_ancestors, - uint64_t limitAncestorCount, - uint64_t limitAncestorSize, - uint64_t limitDescendantCount, - uint64_t limitDescendantSize, + const Limits& limits, std::string &errString) const EXCLUSIVE_LOCKS_REQUIRED(cs); public: @@ -576,8 +581,6 @@ public: const bool m_require_standard; const bool m_full_rbf; - using Limits = kernel::MemPoolLimits; - const Limits m_limits; /** Create a new CTxMemPool. @@ -668,38 +671,41 @@ public: */ void UpdateTransactionsFromBlock(const std::vector<uint256>& vHashesToUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main) LOCKS_EXCLUDED(m_epoch); - /** Try to calculate all in-mempool ancestors of entry. - * (these are all calculated including the tx itself) - * limitAncestorCount = max number of ancestors - * limitAncestorSize = max size of ancestors - * limitDescendantCount = max number of descendants any ancestor can have - * limitDescendantSize = max size of descendants any ancestor can have - * errString = populated with error reason if any limits are hit - * fSearchForParents = whether to search a tx's vin for in-mempool parents, or - * look up parents from mapLinks. Must be true for entries not in the mempool + /** + * Try to calculate all in-mempool ancestors of entry. + * (these are all calculated including the tx itself) + * + * @param[in] entry CTxMemPoolEntry of which all in-mempool ancestors are calculated + * @param[out] setAncestors Will be populated with all mempool ancestors. + * @param[in] limits Maximum number and size of ancestors and descendants + * @param[out] errString Populated with error reason if any limits are hit + * @param[in] fSearchForParents Whether to search a tx's vin for in-mempool parents, or look + * up parents from mapLinks. Must be true for entries not in + * the mempool + * + * @return true if no limits were hit and all in-mempool ancestors were calculated, false + * otherwise */ - bool CalculateMemPoolAncestors(const CTxMemPoolEntry& entry, setEntries& setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string& errString, bool fSearchForParents = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); + bool CalculateMemPoolAncestors(const CTxMemPoolEntry& entry, + setEntries& setAncestors, + const Limits& limits, + std::string& errString, + bool fSearchForParents = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Calculate all in-mempool ancestors of a set of transactions not already in the mempool and * check ancestor and descendant limits. Heuristics are used to estimate the ancestor and * descendant count of all entries if the package were to be added to the mempool. The limits * are applied to the union of all package transactions. For example, if the package has 3 - * transactions and limitAncestorCount = 25, the union of all 3 sets of ancestors (including the + * transactions and limits.ancestor_count = 25, the union of all 3 sets of ancestors (including the * transactions themselves) must be <= 22. * @param[in] package Transaction package being evaluated for acceptance * to mempool. The transactions need not be direct * ancestors/descendants of each other. - * @param[in] limitAncestorCount Max number of txns including ancestors. - * @param[in] limitAncestorSize Max virtual size including ancestors. - * @param[in] limitDescendantCount Max number of txns including descendants. - * @param[in] limitDescendantSize Max virtual size including descendants. + * @param[in] limits Maximum number and size of ancestors and descendants * @param[out] errString Populated with error reason if a limit is hit. */ bool CheckPackageLimits(const Package& package, - uint64_t limitAncestorCount, - uint64_t limitAncestorSize, - uint64_t limitDescendantCount, - uint64_t limitDescendantSize, + const Limits& limits, std::string &errString) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Populate setDescendants with all in-mempool descendants of hash. @@ -825,7 +831,7 @@ private: * mempool but may have child transactions in the mempool, eg during a * chain reorg. * - * @pre CTxMemPool::m_children is correct for the given tx and all + * @pre CTxMemPoolEntry::m_children is correct for the given tx and all * descendants. * @pre cachedDescendants is an accurate cache where each entry has all * descendants of the corresponding key, including those that should diff --git a/src/util/error.cpp b/src/util/error.cpp index 33a35a6d59..390cb6c11b 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -49,6 +49,11 @@ bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBi return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); } +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) +{ + return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); +} + bilingual_str AmountHighWarn(const std::string& optname) { return strprintf(_("%s is set very high!"), optname); diff --git a/src/util/error.h b/src/util/error.h index 0429de651a..27916501f0 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -39,6 +39,8 @@ bilingual_str TransactionErrorString(const TransactionError error); bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); + bilingual_str AmountHighWarn(const std::string& optname); bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 125dbc7f18..84ac2759fa 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -117,6 +117,34 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } +bool Sock::SetNonBlocking() const +{ +#ifdef WIN32 + u_long on{1}; + if (ioctlsocket(m_socket, FIONBIO, &on) == SOCKET_ERROR) { + return false; + } +#else + const int flags{fcntl(m_socket, F_GETFL, 0)}; + if (flags == SOCKET_ERROR) { + return false; + } + if (fcntl(m_socket, F_SETFL, flags | O_NONBLOCK) == SOCKET_ERROR) { + return false; + } +#endif + return true; +} + +bool Sock::IsSelectable() const +{ +#if defined(USE_POLL) || defined(WIN32) + return true; +#else + return m_socket < FD_SETSIZE; +#endif +} + bool Sock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { // We need a `shared_ptr` owning `this` for `WaitMany()`, but don't want @@ -185,10 +213,10 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per SOCKET socket_max{0}; for (const auto& [sock, events] : events_per_sock) { - const auto& s = sock->m_socket; - if (!IsSelectableSocket(s)) { + if (!sock->IsSelectable()) { return false; } + const auto& s = sock->m_socket; if (events.requested & RECV) { FD_SET(s, &recv); } diff --git a/src/util/sock.h b/src/util/sock.h index 38a7dc80d6..7912284904 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -133,6 +133,18 @@ public: */ [[nodiscard]] virtual int GetSockName(sockaddr* name, socklen_t* name_len) const; + /** + * Set the non-blocking option on the socket. + * @return true if set successfully + */ + [[nodiscard]] virtual bool SetNonBlocking() const; + + /** + * Check if the underlying socket can be used for `select(2)` (or the `Wait()` method). + * @return true if selectable + */ + [[nodiscard]] virtual bool IsSelectable() const; + using Event = uint8_t; /** diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index b5ac151374..e28ca8e73a 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -97,8 +97,9 @@ std::vector<Byte> ParseHex(std::string_view str) template std::vector<std::byte> ParseHex(std::string_view); template std::vector<uint8_t> ParseHex(std::string_view); -void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) +bool SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) { + bool valid = false; size_t colon = in.find_last_of(':'); // if a : is found, and it either follows a [...], or no other : is in the string, treat it as port separator bool fHaveColon = colon != in.npos; @@ -109,13 +110,18 @@ void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) if (ParseUInt16(in.substr(colon + 1), &n)) { in = in.substr(0, colon); portOut = n; + valid = (portOut != 0); } + } else { + valid = true; } if (in.size() > 0 && in[0] == '[' && in[in.size() - 1] == ']') { hostOut = in.substr(1, in.size() - 2); } else { hostOut = in; } + + return valid; } std::string EncodeBase64(Span<const unsigned char> input) diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 14867b21b2..94bc6cc2f3 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -88,7 +88,16 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); */ std::string EncodeBase32(std::string_view str, bool pad = true); -void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); +/** + * Splits socket address string into host string and port value. + * Validates port value. + * + * @param[in] in The socket address string to split. + * @param[out] portOut Port-portion of the input, if found and parsable. + * @param[out] hostOut Host-portion of the input, if found. + * @return true if port-portion is absent or within its allowed range, otherwise false + */ +bool SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); // LocaleIndependentAtoi is provided for backwards compatibility reasons. // diff --git a/src/util/system.cpp b/src/util/system.cpp index c3c6cbfef6..d7a0793ea8 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -5,19 +5,6 @@ #include <util/system.h> -#ifdef ENABLE_EXTERNAL_SIGNER -#if defined(__GNUC__) -// Boost 1.78 requires the following workaround. -// See: https://github.com/boostorg/process/issues/235 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wnarrowing" -#endif -#include <boost/process.hpp> -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#endif -#endif // ENABLE_EXTERNAL_SIGNER - #include <chainparamsbase.h> #include <fs.h> #include <sync.h> @@ -935,6 +922,20 @@ static bool GetConfigOptions(std::istream& stream, const std::string& filepath, return true; } +bool IsConfSupported(KeyInfo& key, std::string& error) { + if (key.name == "conf") { + error = "conf cannot be set in the configuration file; use includeconf= if you want to include additional config files"; + return false; + } + if (key.name == "reindex") { + // reindex can be set in a config file but it is strongly discouraged as this will cause the node to reindex on + // every restart. Allow the config but throw a warning + LogPrintf("Warning: reindex=1 is set in the configuration file, which will significantly slow down startup. Consider removing or commenting out this option for better performance, unless there is currently a condition which makes rebuilding the indexes necessary\n"); + return true; + } + return true; +} + bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); @@ -945,6 +946,7 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file for (const std::pair<std::string, std::string>& option : options) { KeyInfo key = InterpretKey(option.first); std::optional<unsigned int> flags = GetArgFlags('-' + key.name); + if (!IsConfSupported(key, error)) return false; if (flags) { std::optional<util::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); if (!value) { @@ -1332,44 +1334,6 @@ void runCommand(const std::string& strCommand) } #endif -UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) -{ -#ifdef ENABLE_EXTERNAL_SIGNER - namespace bp = boost::process; - - UniValue result_json; - bp::opstream stdin_stream; - bp::ipstream stdout_stream; - bp::ipstream stderr_stream; - - if (str_command.empty()) return UniValue::VNULL; - - bp::child c( - str_command, - bp::std_out > stdout_stream, - bp::std_err > stderr_stream, - bp::std_in < stdin_stream - ); - if (!str_std_in.empty()) { - stdin_stream << str_std_in << std::endl; - } - stdin_stream.pipe().close(); - - std::string result; - std::string error; - std::getline(stdout_stream, result); - std::getline(stderr_stream, error); - - c.wait(); - const int n_error = c.exit_code(); - if (n_error) throw std::runtime_error(strprintf("RunCommandParseJSON error: process(%s) returned %d: %s\n", str_command, n_error, error)); - if (!result_json.read(result)) throw std::runtime_error("Unable to parse JSON: " + result); - - return result_json; -#else - throw std::runtime_error("Compiled without external signing support (required for external signing)."); -#endif // ENABLE_EXTERNAL_SIGNER -} void SetupEnvironment() { diff --git a/src/util/system.h b/src/util/system.h index c8e1de6700..29629e547e 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -107,14 +107,6 @@ std::string ShellEscape(const std::string& arg); #if HAVE_SYSTEM void runCommand(const std::string& strCommand); #endif -/** - * Execute a command which returns JSON, and parse the result. - * - * @param str_command The command to execute, including any arguments - * @param str_std_in string to pass to stdin - * @return parsed JSON - */ -UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in=""); /** * Most paths passed as configuration arguments are treated as relative to diff --git a/src/util/time.h b/src/util/time.h index 0ec79bec32..10a581a44c 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -57,6 +57,7 @@ constexpr int64_t count_microseconds(std::chrono::microseconds t) { return t.cou using HoursDouble = std::chrono::duration<double, std::chrono::hours::period>; using SecondsDouble = std::chrono::duration<double, std::chrono::seconds::period>; +using MillisecondsDouble = std::chrono::duration<double, std::chrono::milliseconds::period>; /** * DEPRECATED diff --git a/src/validation.cpp b/src/validation.cpp index fb29ca3ae7..37e68cfe4a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -82,9 +82,6 @@ using node::SnapshotMetadata; using node::UndoReadFromDisk; using node::UnlinkPrunedFiles; -#define MICRO 0.000001 -#define MILLI 0.001 - /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** Time to wait between writing blocks/block index to disk. */ @@ -424,11 +421,13 @@ namespace { class MemPoolAccept { public: - explicit MemPoolAccept(CTxMemPool& mempool, Chainstate& active_chainstate) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&active_chainstate.CoinsTip(), m_pool), m_active_chainstate(active_chainstate), - m_limit_ancestors(m_pool.m_limits.ancestor_count), - m_limit_ancestor_size(m_pool.m_limits.ancestor_size_vbytes), - m_limit_descendants(m_pool.m_limits.descendant_count), - m_limit_descendant_size(m_pool.m_limits.descendant_size_vbytes) { + explicit MemPoolAccept(CTxMemPool& mempool, Chainstate& active_chainstate) : + m_pool(mempool), + m_view(&m_dummy), + m_viewmempool(&active_chainstate.CoinsTip(), m_pool), + m_active_chainstate(active_chainstate), + m_limits{m_pool.m_limits} + { } // We put the arguments we're handed into a struct, so we can pass them @@ -660,13 +659,7 @@ private: Chainstate& m_active_chainstate; - // The package limits in effect at the time of invocation. - const size_t m_limit_ancestors; - const size_t m_limit_ancestor_size; - // These may be modified while evaluating a transaction (eg to account for - // in-mempool conflicts; see below). - size_t m_limit_descendants; - size_t m_limit_descendant_size; + CTxMemPool::Limits m_limits; /** Whether the transaction(s) would replace any mempool transactions. If so, RBF rules apply. */ bool m_rbf{false}; @@ -879,12 +872,12 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) assert(ws.m_iters_conflicting.size() == 1); CTxMemPool::txiter conflict = *ws.m_iters_conflicting.begin(); - m_limit_descendants += 1; - m_limit_descendant_size += conflict->GetSizeWithDescendants(); + m_limits.descendant_count += 1; + m_limits.descendant_size_vbytes += conflict->GetSizeWithDescendants(); } std::string errString; - if (!m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, m_limit_descendant_size, errString)) { + if (!m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, m_limits, errString)) { ws.m_ancestors.clear(); // If CalculateMemPoolAncestors fails second time, we want the original error string. std::string dummy_err_string; @@ -899,8 +892,16 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // to be secure by simply only having two immediately-spendable // outputs - one for each counterparty. For more info on the uses for // this, see https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html + CTxMemPool::Limits cpfp_carve_out_limits{ + .ancestor_count = 2, + .ancestor_size_vbytes = m_limits.ancestor_size_vbytes, + .descendant_count = m_limits.descendant_count + 1, + .descendant_size_vbytes = m_limits.descendant_size_vbytes + EXTRA_DESCENDANT_TX_SIZE_LIMIT, + }; if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || - !m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { + !m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, + cpfp_carve_out_limits, + dummy_err_string)) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", errString); } } @@ -976,8 +977,7 @@ bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txn { return !m_pool.exists(GenTxid::Txid(tx->GetHash()));})); std::string err_string; - if (!m_pool.CheckPackageLimits(txns, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, - m_limit_descendant_size, err_string)) { + if (!m_pool.CheckPackageLimits(txns, m_limits, err_string)) { // This is a package-wide error, separate from an individual transaction error. return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); } @@ -1121,9 +1121,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& // Re-calculate mempool ancestors to call addUnchecked(). They may have changed since the // last calculation done in PreChecks, since package ancestors have already been submitted. std::string unused_err_string; - if(!m_pool.CalculateMemPoolAncestors(*ws.m_entry, ws.m_ancestors, m_limit_ancestors, - m_limit_ancestor_size, m_limit_descendants, - m_limit_descendant_size, unused_err_string)) { + if(!m_pool.CalculateMemPoolAncestors(*ws.m_entry, ws.m_ancestors, m_limits, unused_err_string)) { results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); // Since PreChecks() and PackageMempoolChecks() both enforce limits, this should never fail. Assume(false); @@ -1416,7 +1414,7 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - const CChainParams& chainparams{active_chainstate.m_params}; + const CChainParams& chainparams{active_chainstate.m_chainman.GetParams()}; assert(active_chainstate.GetMempool() != nullptr); CTxMemPool& pool{*active_chainstate.GetMempool()}; @@ -1446,7 +1444,7 @@ PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxM assert(std::all_of(package.cbegin(), package.cend(), [](const auto& tx){return tx != nullptr;})); std::vector<COutPoint> coins_to_uncache; - const CChainParams& chainparams = active_chainstate.m_params; + const CChainParams& chainparams = active_chainstate.m_chainman.GetParams(); const auto result = [&]() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); if (test_accept) { @@ -1504,7 +1502,6 @@ Chainstate::Chainstate( std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), m_blockman(blockman), - m_params(chainman.GetParams()), m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} @@ -1515,7 +1512,7 @@ void Chainstate::InitCoinsDB( fs::path leveldb_name) { if (m_from_snapshot_blockhash) { - leveldb_name += "_" + m_from_snapshot_blockhash->ToString(); + leveldb_name += node::SNAPSHOT_CHAINSTATE_SUFFIX; } m_coins_views = std::make_unique<CoinsViews>( @@ -1840,11 +1837,21 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn return DISCONNECT_FAILED; } + // Ignore blocks that contain transactions which are 'overwritten' by later transactions, + // unless those are already completely spent. + // See https://github.com/bitcoin/bitcoin/issues/22596 for additional information. + // Note: the blocks specified here are different than the ones used in ConnectBlock because DisconnectBlock + // unwinds the blocks in reverse. As a result, the inconsistency is not discovered until the earlier + // blocks with the duplicate coinbase transactions are disconnected. + bool fEnforceBIP30 = !((pindex->nHeight==91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight==91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))); + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); bool is_coinbase = tx.IsCoinBase(); + bool is_bip30_exception = (is_coinbase && !fEnforceBIP30); // Check that all outputs are available and match the outputs in the block itself // exactly. @@ -1854,7 +1861,9 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn Coin coin; bool is_spent = view.SpendCoin(out, &coin); if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { - fClean = false; // transaction output mismatch + if (!is_bip30_exception) { + fClean = false; // transaction output mismatch + } } } } @@ -1965,14 +1974,14 @@ static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Ch } -static int64_t nTimeCheck = 0; -static int64_t nTimeForks = 0; -static int64_t nTimeConnect = 0; -static int64_t nTimeVerify = 0; -static int64_t nTimeUndo = 0; -static int64_t nTimeIndex = 0; -static int64_t nTimeTotal = 0; -static int64_t nBlocksTotal = 0; +static SteadyClock::duration time_check{}; +static SteadyClock::duration time_forks{}; +static SteadyClock::duration time_connect{}; +static SteadyClock::duration time_verify{}; +static SteadyClock::duration time_undo{}; +static SteadyClock::duration time_index{}; +static SteadyClock::duration time_total{}; +static int64_t num_blocks_total = 0; /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() @@ -1986,7 +1995,8 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, uint256 block_hash{block.GetHash()}; assert(*pindex->phashBlock == block_hash); - int64_t nTimeStart = GetTimeMicros(); + const auto time_start{SteadyClock::now()}; + const CChainParams& params{m_chainman.GetParams()}; // Check it again in case a previous version let a bad block in // NOTE: We don't currently (re-)invoke ContextualCheckBlock() or @@ -2001,7 +2011,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // is enforced in ContextualCheckBlockHeader(); we wouldn't want to // re-enforce that rule here (at least until we make it impossible for // m_adjusted_time_callback() to go backward). - if (!CheckBlock(block, state, m_params.GetConsensus(), !fJustCheck, !fJustCheck)) { + if (!CheckBlock(block, state, params.GetConsensus(), !fJustCheck, !fJustCheck)) { if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) { // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware @@ -2015,11 +2025,11 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, uint256 hashPrevBlock = pindex->pprev == nullptr ? uint256() : pindex->pprev->GetBlockHash(); assert(hashPrevBlock == view.GetBestBlock()); - nBlocksTotal++; + num_blocks_total++; // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) - if (block_hash == m_params.GetConsensus().hashGenesisBlock) { + if (block_hash == params.GetConsensus().hashGenesisBlock) { if (!fJustCheck) view.SetBestBlock(pindex->GetBlockHash()); return true; @@ -2051,13 +2061,17 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // artificially set the default assumed verified block further back. // The test against nMinimumChainWork prevents the skipping when denied access to any chain at // least as good as the expected chain. - fScriptChecks = (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, m_params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); + fScriptChecks = (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); } } } - int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; - LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime1 - nTimeStart), nTimeCheck * MICRO, nTimeCheck * MILLI / nBlocksTotal); + const auto time_1{SteadyClock::now()}; + time_check += time_1 - time_start; + LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_1 - time_start), + Ticks<SecondsDouble>(time_check), + Ticks<MillisecondsDouble>(time_check) / num_blocks_total); // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. @@ -2128,9 +2142,9 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // post BIP34 before approximately height 486,000,000. After block // 1,983,702 testnet3 starts doing unnecessary BIP30 checking again. assert(pindex->pprev); - CBlockIndex* pindexBIP34height = pindex->pprev->GetAncestor(m_params.GetConsensus().BIP34Height); + CBlockIndex* pindexBIP34height = pindex->pprev->GetAncestor(params.GetConsensus().BIP34Height); //Only continue to enforce if we're below BIP34 activation height or the block hash at that height doesn't correspond. - fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == m_params.GetConsensus().BIP34Hash)); + fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == params.GetConsensus().BIP34Hash)); // TODO: Remove BIP30 checking from block height 1,983,702 on, once we have a // consensus change that ensures coinbases at those heights cannot @@ -2155,8 +2169,12 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // Get the script flags for this block unsigned int flags{GetBlockScriptFlags(*pindex, m_chainman)}; - int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; - LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); + const auto time_2{SteadyClock::now()}; + time_forks += time_2 - time_1; + LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_2 - time_1), + Ticks<SecondsDouble>(time_forks), + Ticks<MillisecondsDouble>(time_forks) / num_blocks_total); CBlockUndo blockundo; @@ -2240,10 +2258,15 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, } UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); } - int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; - LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); + const auto time_3{SteadyClock::now()}; + time_connect += time_3 - time_2; + LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), + Ticks<MillisecondsDouble>(time_3 - time_2), Ticks<MillisecondsDouble>(time_3 - time_2) / block.vtx.size(), + nInputs <= 1 ? 0 : Ticks<MillisecondsDouble>(time_3 - time_2) / (nInputs - 1), + Ticks<SecondsDouble>(time_connect), + Ticks<MillisecondsDouble>(time_connect) / num_blocks_total); - CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, m_params.GetConsensus()); + CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, params.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) { LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); @@ -2253,18 +2276,27 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, LogPrintf("ERROR: %s: CheckQueue failed\n", __func__); return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "block-validation-failed"); } - int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; - LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); + const auto time_4{SteadyClock::now()}; + time_verify += time_4 - time_2; + LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, + Ticks<MillisecondsDouble>(time_4 - time_2), + nInputs <= 1 ? 0 : Ticks<MillisecondsDouble>(time_4 - time_2) / (nInputs - 1), + Ticks<SecondsDouble>(time_verify), + Ticks<MillisecondsDouble>(time_verify) / num_blocks_total); if (fJustCheck) return true; - if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { + if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, params)) { return false; } - int64_t nTime5 = GetTimeMicros(); nTimeUndo += nTime5 - nTime4; - LogPrint(BCLog::BENCH, " - Write undo data: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeUndo * MICRO, nTimeUndo * MILLI / nBlocksTotal); + const auto time_5{SteadyClock::now()}; + time_undo += time_5 - time_4; + LogPrint(BCLog::BENCH, " - Write undo data: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_5 - time_4), + Ticks<SecondsDouble>(time_undo), + Ticks<MillisecondsDouble>(time_undo) / num_blocks_total); if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); @@ -2274,8 +2306,12 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); - int64_t nTime6 = GetTimeMicros(); nTimeIndex += nTime6 - nTime5; - LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); + const auto time_6{SteadyClock::now()}; + time_index += time_6 - time_5; + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_6 - time_5), + Ticks<SecondsDouble>(time_index), + Ticks<MillisecondsDouble>(time_index) / num_blocks_total); TRACE6(validation, block_connected, block_hash.data(), @@ -2283,7 +2319,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, block.vtx.size(), nInputs, nSigOpsCost, - nTime5 - nTimeStart // in microseconds (µs) + time_5 - time_start // in microseconds (µs) ); return true; @@ -2370,7 +2406,7 @@ bool Chainstate::FlushStateToDisk( } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, m_chainman.GetParams().PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { @@ -2524,13 +2560,15 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) AssertLockHeld(::cs_main); const auto& coins_tip = this->CoinsTip(); + const CChainParams& params{m_chainman.GetParams()}; + // The remainder of the function isn't relevant if we are not acting on // the active chainstate, so return if need be. if (this != &m_chainman.ActiveChainstate()) { // Only log every so often so that we don't bury log messages at the tip. constexpr int BACKGROUND_LOG_INTERVAL = 2000; if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { - UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "[background validation] ", ""); + UpdateTipLog(coins_tip, pindexNew, params, __func__, "[background validation] ", ""); } return; } @@ -2551,7 +2589,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); - ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache.at(bit)); + ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2562,7 +2600,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) } } } - UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "", warning_messages.original); + UpdateTipLog(coins_tip, pindexNew, params, __func__, "", warning_messages.original); } /** Disconnect m_chain's tip. @@ -2586,11 +2624,11 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra // Read block from disk. std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; - if (!ReadBlockFromDisk(block, pindexDelete, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindexDelete, m_chainman.GetConsensus())) { return error("DisconnectTip(): Failed to read block"); } // Apply the block atomically to the chain state. - int64_t nStart = GetTimeMicros(); + const auto time_start{SteadyClock::now()}; { CCoinsViewCache view(&CoinsTip()); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); @@ -2599,7 +2637,8 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra bool flushed = view.Flush(); assert(flushed); } - LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); + LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", + Ticks<MillisecondsDouble>(SteadyClock::now() - time_start)); { // Prune locks that began at or after the tip should be moved backward so they get a chance to reorg @@ -2639,11 +2678,11 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra return true; } -static int64_t nTimeReadFromDiskTotal = 0; -static int64_t nTimeConnectTotal = 0; -static int64_t nTimeFlush = 0; -static int64_t nTimeChainState = 0; -static int64_t nTimePostConnect = 0; +static SteadyClock::duration time_read_from_disk_total{}; +static SteadyClock::duration time_connect_total{}; +static SteadyClock::duration time_flush{}; +static SteadyClock::duration time_chainstate{}; +static SteadyClock::duration time_post_connect{}; struct PerBlockConnectTrace { CBlockIndex* pindex = nullptr; @@ -2698,11 +2737,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. - int64_t nTime1 = GetTimeMicros(); + const auto time_1{SteadyClock::now()}; std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_chainman.GetConsensus())) { return AbortNode(state, "Failed to read block"); } pthisBlock = pblockNew; @@ -2712,9 +2751,13 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, } const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. - int64_t nTime2 = GetTimeMicros(); nTimeReadFromDiskTotal += nTime2 - nTime1; - int64_t nTime3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDiskTotal * MICRO, nTimeReadFromDiskTotal * MILLI / nBlocksTotal); + const auto time_2{SteadyClock::now()}; + time_read_from_disk_total += time_2 - time_1; + SteadyClock::time_point time_3; + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_2 - time_1), + Ticks<SecondsDouble>(time_read_from_disk_total), + Ticks<MillisecondsDouble>(time_read_from_disk_total) / num_blocks_total); { CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); @@ -2724,20 +2767,32 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, InvalidBlockFound(pindexNew, state); return error("%s: ConnectBlock %s failed, %s", __func__, pindexNew->GetBlockHash().ToString(), state.ToString()); } - nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; - assert(nBlocksTotal > 0); - LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime3 - nTime2) * MILLI, nTimeConnectTotal * MICRO, nTimeConnectTotal * MILLI / nBlocksTotal); + time_3 = SteadyClock::now(); + time_connect_total += time_3 - time_2; + assert(num_blocks_total > 0); + LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_3 - time_2), + Ticks<SecondsDouble>(time_connect_total), + Ticks<MillisecondsDouble>(time_connect_total) / num_blocks_total); bool flushed = view.Flush(); assert(flushed); } - int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; - LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); + const auto time_4{SteadyClock::now()}; + time_flush += time_4 - time_3; + LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_4 - time_3), + Ticks<SecondsDouble>(time_flush), + Ticks<MillisecondsDouble>(time_flush) / num_blocks_total); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) { return false; } - int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; - LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); + const auto time_5{SteadyClock::now()}; + time_chainstate += time_5 - time_4; + LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_5 - time_4), + Ticks<SecondsDouble>(time_chainstate), + Ticks<MillisecondsDouble>(time_chainstate) / num_blocks_total); // Remove conflicting transactions from the mempool.; if (m_mempool) { m_mempool->removeForBlock(blockConnecting.vtx, pindexNew->nHeight); @@ -2747,9 +2802,17 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, m_chain.SetTip(*pindexNew); UpdateTip(pindexNew); - int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; - LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); - LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime1) * MILLI, nTimeTotal * MICRO, nTimeTotal * MILLI / nBlocksTotal); + const auto time_6{SteadyClock::now()}; + time_post_connect += time_6 - time_5; + time_total += time_6 - time_1; + LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_6 - time_5), + Ticks<SecondsDouble>(time_post_connect), + Ticks<MillisecondsDouble>(time_post_connect) / num_blocks_total); + LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs (%.2fms/blk)]\n", + Ticks<MillisecondsDouble>(time_6 - time_1), + Ticks<SecondsDouble>(time_total), + Ticks<MillisecondsDouble>(time_total) / num_blocks_total); connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; @@ -3786,7 +3849,9 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV if (pindex->nChainWork < nMinimumChainWork) return true; } - if (!CheckBlock(block, state, m_params.GetConsensus()) || + const CChainParams& params{m_chainman.GetParams()}; + + if (!CheckBlock(block, state, params.GetConsensus()) || !ContextualCheckBlock(block, state, m_chainman, pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; @@ -3803,7 +3868,7 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, params, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3947,7 +4012,7 @@ bool Chainstate::LoadChainTip() tip->GetBlockHash().ToString(), m_chain.Height(), FormatISO8601DateTime(tip->GetBlockTime()), - GuessVerificationProgress(m_params.TxData(), tip)); + GuessVerificationProgress(m_chainman.GetParams().TxData(), tip)); return true; } @@ -4083,7 +4148,7 @@ bool Chainstate::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& in AssertLockHeld(cs_main); // TODO: merge with ConnectBlock CBlock block; - if (!ReadBlockFromDisk(block, pindex, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindex, m_chainman.GetConsensus())) { return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4135,7 +4200,7 @@ bool Chainstate::ReplayBlocks() while (pindexOld != pindexFork) { if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; - if (!ReadBlockFromDisk(block, pindexOld, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindexOld, m_chainman.GetConsensus())) { return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); @@ -4287,16 +4352,18 @@ bool Chainstate::LoadGenesisBlock() { LOCK(cs_main); + const CChainParams& params{m_chainman.GetParams()}; + // Check whether we're already initialized by checking for genesis in // m_blockman.m_block_index. Note that we can't use m_chain here, since it is // set based on the coins db, not the block index db, which is the only // thing loaded at this point. - if (m_blockman.m_block_index.count(m_params.GenesisBlock().GetHash())) + if (m_blockman.m_block_index.count(params.GenesisBlock().GetHash())) return true; try { - const CBlock& block = m_params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, m_params, nullptr)}; + const CBlock& block = params.GenesisBlock(); + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, params, nullptr)}; if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } @@ -4320,6 +4387,7 @@ void Chainstate::LoadExternalBlockFile( assert(!dbp == !blocks_with_unknown_parent); const auto start{SteadyClock::now()}; + const CChainParams& params{m_chainman.GetParams()}; int nLoaded = 0; try { @@ -4336,10 +4404,10 @@ void Chainstate::LoadExternalBlockFile( try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; - blkdat.FindByte(m_params.MessageStart()[0]); + blkdat.FindByte(params.MessageStart()[0]); nRewind = blkdat.GetPos() + 1; blkdat >> buf; - if (memcmp(buf, m_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { + if (memcmp(buf, params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { continue; } // read size @@ -4365,7 +4433,7 @@ void Chainstate::LoadExternalBlockFile( { LOCK(cs_main); // detect out of order blocks, and store them for later - if (hash != m_params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { + if (hash != params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp && blocks_with_unknown_parent) { @@ -4384,13 +4452,13 @@ void Chainstate::LoadExternalBlockFile( if (state.IsError()) { break; } - } else if (hash != m_params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { + } else if (hash != params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight); } } // Activate the genesis block so normal node progress can continue - if (hash == m_params.GetConsensus().hashGenesisBlock) { + if (hash == params.GetConsensus().hashGenesisBlock) { BlockValidationState state; if (!ActivateBestChain(state, nullptr)) { break; @@ -4411,7 +4479,7 @@ void Chainstate::LoadExternalBlockFile( while (range.first != range.second) { std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); - if (ReadBlockFromDisk(*pblockrecursive, it->second, m_params.GetConsensus())) { + if (ReadBlockFromDisk(*pblockrecursive, it->second, params.GetConsensus())) { LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); @@ -4427,7 +4495,18 @@ void Chainstate::LoadExternalBlockFile( } } } catch (const std::exception& e) { - LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + // historical bugs added extra data to the block files that does not deserialize cleanly. + // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process. + // the code that reads the block files deals with invalid data by simply ignoring it. + // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly + // and passes all of the other block validation checks dealing with POW and the merkle root, etc... + // we merely note with this informational log message when unexpected data is encountered. + // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but + // less likely scenarios. we don't have enough information to tell a difference here. + // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator + // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and + // perhaps ordered, block files for later reindexing. + LogPrint(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what()); } } } catch (const std::runtime_error& e) { @@ -4511,7 +4590,7 @@ void Chainstate::CheckBlockIndex() // Begin: actual consistency checks. if (pindex->pprev == nullptr) { // Genesis block checks. - assert(pindex->GetBlockHash() == m_params.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. + assert(pindex->GetBlockHash() == m_chainman.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. assert(pindex == m_chain.Genesis()); // The current active chain's genesis block must be this block. } if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) @@ -4744,28 +4823,15 @@ std::vector<Chainstate*> ChainstateManager::GetAll() return out; } -Chainstate& ChainstateManager::InitializeChainstate( - CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash) +Chainstate& ChainstateManager::InitializeChainstate(CTxMemPool* mempool) { AssertLockHeld(::cs_main); - bool is_snapshot = snapshot_blockhash.has_value(); - std::unique_ptr<Chainstate>& to_modify = - is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; - - if (to_modify) { - throw std::logic_error("should not be overwriting a chainstate"); - } - to_modify.reset(new Chainstate(mempool, m_blockman, *this, snapshot_blockhash)); + assert(!m_ibd_chainstate); + assert(!m_active_chainstate); - // Snapshot chainstates and initial IBD chaintates always become active. - if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { - LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); - m_active_chainstate = to_modify.get(); - } else { - throw std::logic_error("unexpected chainstate activation"); - } - - return *to_modify; + m_ibd_chainstate = std::make_unique<Chainstate>(mempool, m_blockman, *this); + m_active_chainstate = m_ibd_chainstate.get(); + return *m_active_chainstate; } const AssumeutxoData* ExpectedAssumeutxo( @@ -4780,6 +4846,46 @@ const AssumeutxoData* ExpectedAssumeutxo( return nullptr; } +static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + AssertLockHeld(::cs_main); + + if (is_snapshot) { + fs::path base_blockhash_path = db_path / node::SNAPSHOT_BLOCKHASH_FILENAME; + + if (fs::exists(base_blockhash_path)) { + bool removed = fs::remove(base_blockhash_path); + if (!removed) { + LogPrintf("[snapshot] failed to remove file %s\n", + fs::PathToString(base_blockhash_path)); + } + } else { + LogPrintf("[snapshot] snapshot chainstate dir being removed lacks %s file\n", + fs::PathToString(node::SNAPSHOT_BLOCKHASH_FILENAME)); + } + } + + std::string path_str = fs::PathToString(db_path); + LogPrintf("Removing leveldb dir at %s\n", path_str); + + // We have to destruct before this call leveldb::DB in order to release the db + // lock, otherwise `DestroyDB` will fail. See `leveldb::~DBImpl()`. + const bool destroyed = dbwrapper::DestroyDB(path_str, {}).ok(); + + if (!destroyed) { + LogPrintf("error: leveldb DestroyDB call failed on %s\n", path_str); + } + + // Datadir should be removed from filesystem; otherwise initialization may detect + // it on subsequent statups and get confused. + // + // If the base_blockhash_path removal above fails in the case of snapshot + // chainstates, this will return false since leveldb won't remove a non-empty + // directory. + return destroyed && !fs::exists(db_path); +} + bool ChainstateManager::ActivateSnapshot( AutoFile& coins_file, const SnapshotMetadata& metadata, @@ -4837,11 +4943,34 @@ bool ChainstateManager::ActivateSnapshot( static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC)); } - const bool snapshot_ok = this->PopulateAndValidateSnapshot( + bool snapshot_ok = this->PopulateAndValidateSnapshot( *snapshot_chainstate, coins_file, metadata); + // If not in-memory, persist the base blockhash for use during subsequent + // initialization. + if (!in_memory) { + LOCK(::cs_main); + if (!node::WriteSnapshotBaseBlockhash(*snapshot_chainstate)) { + snapshot_ok = false; + } + } if (!snapshot_ok) { - WITH_LOCK(::cs_main, this->MaybeRebalanceCaches()); + LOCK(::cs_main); + this->MaybeRebalanceCaches(); + + // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir + // has been created, so only attempt removal if we got that far. + if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) { + // We have to destruct leveldb::DB in order to release the db lock, otherwise + // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`. + // Destructing the chainstate (and so resetting the coinsviews object) does this. + snapshot_chainstate.reset(); + bool removed = DeleteCoinsDBFromDisk(*snapshot_datadir, /*is_snapshot=*/true); + if (!removed) { + AbortNode(strprintf("Failed to remove snapshot chainstate dir (%s). " + "Manually remove it before restarting.\n", fs::PathToString(*snapshot_datadir))); + } + } return false; } @@ -5115,6 +5244,13 @@ void ChainstateManager::MaybeRebalanceCaches() } } +void ChainstateManager::ResetChainstates() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; +} + ChainstateManager::~ChainstateManager() { LOCK(::cs_main); @@ -5126,3 +5262,31 @@ ChainstateManager::~ChainstateManager() i.clear(); } } + +bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) +{ + assert(!m_snapshot_chainstate); + std::optional<fs::path> path = node::FindSnapshotChainstateDir(); + if (!path) { + return false; + } + std::optional<uint256> base_blockhash = node::ReadSnapshotBaseBlockhash(*path); + if (!base_blockhash) { + return false; + } + LogPrintf("[snapshot] detected active snapshot chainstate (%s) - loading\n", + fs::PathToString(*path)); + + this->ActivateExistingSnapshot(mempool, *base_blockhash); + return true; +} + +Chainstate& ChainstateManager::ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) +{ + assert(!m_snapshot_chainstate); + m_snapshot_chainstate = + std::make_unique<Chainstate>(mempool, m_blockman, *this, base_blockhash); + LogPrintf("[snapshot] switching active chainstate to %s\n", m_snapshot_chainstate->ToString()); + m_active_chainstate = m_snapshot_chainstate.get(); + return *m_snapshot_chainstate; +} diff --git a/src/validation.h b/src/validation.h index c882eac408..fb6d59f92e 100644 --- a/src/validation.h +++ b/src/validation.h @@ -472,10 +472,6 @@ public: //! Chainstate instances. node::BlockManager& m_blockman; - /** Chain parameters for this chainstate */ - /* TODO: replace with m_chainman.GetParams() */ - const CChainParams& m_params; - //! The chainstate manager that owns this chainstate. The reference is //! necessary so that this instance can check whether it is the active //! chainstate within deeply nested method calls. @@ -926,17 +922,11 @@ public: //! coins databases. This will be split somehow across chainstates. int64_t m_total_coinsdb_cache{0}; - //! Instantiate a new chainstate and assign it based upon whether it is - //! from a snapshot. + //! Instantiate a new chainstate. //! //! @param[in] mempool The mempool to pass to the chainstate // constructor - //! @param[in] snapshot_blockhash If given, signify that this chainstate - //! is based on a snapshot. - Chainstate& InitializeChainstate( - CTxMemPool* mempool, - const std::optional<uint256>& snapshot_blockhash = std::nullopt) - LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + Chainstate& InitializeChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. std::vector<Chainstate*> GetAll(); @@ -1050,6 +1040,17 @@ public: * information. */ void ReportHeadersPresync(const arith_uint256& work, int64_t height, int64_t timestamp); + //! When starting up, search the datadir for a chainstate based on a UTXO + //! snapshot that is in the process of being validated. + bool DetectSnapshotChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + void ResetChainstates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Switch the active chainstate to one based on a UTXO snapshot that was loaded + //! previously. + Chainstate& ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + ~ChainstateManager(); }; diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 8babbb4298..1206a428fc 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -293,10 +293,7 @@ RPCHelpMan importaddress() if (fRescan) { RescanWallet(*pwallet, reserver); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); } return UniValue::VNULL; @@ -474,10 +471,7 @@ RPCHelpMan importpubkey() if (fRescan) { RescanWallet(*pwallet, reserver); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); } return UniValue::VNULL; @@ -1406,10 +1400,7 @@ RPCHelpMan importmulti() } if (fRescan && fRunScan && requests.size()) { int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); @@ -1700,10 +1691,7 @@ RPCHelpMan importdescriptors() // Rescan the blockchain using the lowest timestamp if (rescan) { int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index ebf694157b..6cb33fc11e 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1379,7 +1379,7 @@ RPCHelpMan sendall() throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not available. UTXO (%s:%d) was already spent.", input.prevout.hash.ToString(), input.prevout.n)); } const CWalletTx* tx{pwallet->GetWalletTx(input.prevout.hash)}; - if (!tx || pwallet->IsMine(tx->tx->vout[input.prevout.n]) != (coin_control.fAllowWatchOnly ? ISMINE_ALL : ISMINE_SPENDABLE)) { + if (!tx || input.prevout.n >= tx->tx->vout.size() || !(pwallet->IsMine(tx->tx->vout[input.prevout.n]) & (coin_control.fAllowWatchOnly ? ISMINE_ALL : ISMINE_SPENDABLE))) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not found. UTXO (%s:%d) is not part of wallet.", input.prevout.hash.ToString(), input.prevout.n)); } total_input_value += tx->tx->vout[input.prevout.n].nValue; diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 41654579c6..39afb79600 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -28,6 +28,9 @@ util::Result<CTxDestination> LegacyScriptPubKeyMan::GetNewDestination(const Outp } assert(type != OutputType::BECH32M); + // Fill-up keypool if needed + TopUp(); + LOCK(cs_KeyStore); // Generate a new key that is added to wallet @@ -304,6 +307,9 @@ util::Result<CTxDestination> LegacyScriptPubKeyMan::GetReservedDestination(const return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } + // Fill-up keypool if needed + TopUp(); + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } @@ -586,7 +592,7 @@ bool LegacyScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sig // or solving information, even if not able to sign fully. return true; } else { - // If, given the stuff in sigdata, we could make a valid sigature, then we can provide for this script + // If, given the stuff in sigdata, we could make a valid signature, then we can provide for this script ProduceSignature(*this, DUMMY_SIGNATURE_CREATOR, script, sigdata); if (!sigdata.signatures.empty()) { // If we could make signatures, make sure we have a private key to actually make a signature @@ -1983,7 +1989,7 @@ util::Result<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const std::optional<OutputType> desc_addr_type = m_wallet_descriptor.descriptor->GetOutputType(); assert(desc_addr_type); if (type != *desc_addr_type) { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); } TopUp(); @@ -2001,11 +2007,8 @@ util::Result<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const } CTxDestination dest; - std::optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); - if (out_script_type && out_script_type == type) { - ExtractDestination(scripts_temp[0], dest); - } else { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + if (!ExtractDestination(scripts_temp[0], dest)) { + return util::Error{_("Error: Cannot extract destination from the generated scriptpubkey")}; // shouldn't happen } m_wallet_descriptor.next_index++; WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); diff --git a/src/wallet/test/feebumper_tests.cpp b/src/wallet/test/feebumper_tests.cpp index 6add86dc3d..2480a9b4e1 100644 --- a/src/wallet/test/feebumper_tests.cpp +++ b/src/wallet/test/feebumper_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php. +#include <consensus/validation.h> +#include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> #include <util/strencodings.h> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 5889de2e36..671c432b10 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -21,6 +21,7 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> +#include <random.h> #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> @@ -124,7 +125,7 @@ bool RemoveWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet interfaces::Chain& chain = wallet->chain(); std::string name = wallet->GetName(); - // Unregister with the validation interface which also drops shared ponters. + // Unregister with the validation interface which also drops shared pointers. wallet->m_chain_notifications_handler.reset(); LOCK(context.wallets_mutex); std::vector<std::shared_ptr<CWallet>>::iterator i = std::find(context.wallets.begin(), context.wallets.end(), wallet); @@ -1903,11 +1904,29 @@ std::set<uint256> CWallet::GetTxConflicts(const CWalletTx& wtx) const return result; } +bool CWallet::ShouldResend() const +{ + // Don't attempt to resubmit if the wallet is configured to not broadcast + if (!fBroadcastTransactions) return false; + + // During reindex, importing and IBD, old wallet transactions become + // unconfirmed. Don't resend them as that would spam other nodes. + // We only allow forcing mempool submission when not relaying to avoid this spam. + if (!chain().isReadyToBroadcast()) return false; + + // Do this infrequently and randomly to avoid giving away + // that these are our transactions. + if (GetTime() < m_next_resend) return false; + + return true; +} + +int64_t CWallet::GetDefaultNextResend() { return GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); } + // Resubmit transactions from the wallet to the mempool, optionally asking the // mempool to relay them. On startup, we will do this for all unconfirmed // transactions but will not ask the mempool to relay them. We do this on startup -// to ensure that our own mempool is aware of our transactions, and to also -// initialize m_next_resend so that the actual rebroadcast is scheduled. There +// to ensure that our own mempool is aware of our transactions. There // is a privacy side effect here as not broadcasting on startup also means that we won't // inform the world of our wallet's state, particularly if the wallet (or node) is not // yet synced. @@ -1934,17 +1953,6 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) // even if forcing. if (!fBroadcastTransactions) return; - // During reindex, importing and IBD, old wallet transactions become - // unconfirmed. Don't resend them as that would spam other nodes. - // We only allow forcing mempool submission when not relaying to avoid this spam. - if (!force && relay && !chain().isReadyToBroadcast()) return; - - // Do this infrequently and randomly to avoid giving away - // that these are our transactions. - if (!force && GetTime() < m_next_resend) return; - // resend 12-36 hours from now, ~1 day on average. - m_next_resend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); - int submitted_tx_count = 0; { // cs_wallet scope @@ -1957,7 +1965,7 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) // Only rebroadcast unconfirmed txs if (!wtx.isUnconfirmed()) continue; - // attempt to rebroadcast all txes more than 5 minutes older than + // Attempt to rebroadcast all txes more than 5 minutes older than // the last block, or all txs if forcing. if (!force && wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; to_submit.insert(&wtx); @@ -1979,7 +1987,9 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) void MaybeResendWalletTxs(WalletContext& context) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { + if (!pwallet->ShouldResend()) continue; pwallet->ResubmitWalletTransactions(/*relay=*/true, /*force=*/false); + pwallet->SetNextResend(); } } @@ -2376,7 +2386,6 @@ util::Result<CTxDestination> CWallet::GetNewDestination(const OutputType type, c return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - spk_man->TopUp(); auto op_dest = spk_man->GetNewDestination(type); if (op_dest) { SetAddressBook(*op_dest, label, "receive"); @@ -2470,10 +2479,7 @@ util::Result<CTxDestination> ReserveDestination::GetReservedDestination(bool int return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - if (nIndex == -1) - { - m_spk_man->TopUp(); - + if (nIndex == -1) { CKeyPool keypool; auto op_address = m_spk_man->GetReservedDestination(type, internal, nIndex, keypool); if (!op_address) return op_address; @@ -3111,12 +3117,14 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. - error = chain.hasAssumedValidChain() ? - _( - "Assumed-valid: last wallet synchronisation goes beyond " - "available block data. You need to wait for the background " - "validation chain to download more blocks.") : - _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); + error = chain.havePruned() ? + _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)") : + strprintf(_( + "Error loading wallet. Wallet requires blocks to be downloaded, " + "and software does not currently support loading wallets while " + "blocks are being downloaded out of order when using assumeutxo " + "snapshots. Wallet should be able to load successfully after " + "node sync reaches height %s"), block_height); return false; } } @@ -3196,14 +3204,12 @@ bool CWallet::UpgradeWallet(int version, bilingual_str& error) void CWallet::postInitProcess() { - LOCK(cs_wallet); - // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); // Update wallet transactions with current mempool transactions. - chain().requestMempoolTransactions(*this); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); } bool CWallet::BackupWallet(const std::string& strDest) const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6a1b76505c..352f43ef99 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -195,7 +195,7 @@ public: util::Result<CTxDestination> GetReservedDestination(bool internal); //! Return reserved address void ReturnDestination(); - //! Keep the address. Do not return it's key to the keypool when this object goes out of scope + //! Keep the address. Do not return its key to the keypool when this object goes out of scope void KeepDestination(); }; @@ -250,7 +250,7 @@ private: int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; /** The next scheduled rebroadcast of wallet transactions. */ - std::atomic<int64_t> m_next_resend{}; + int64_t m_next_resend{GetDefaultNextResend()}; /** Whether this wallet will submit newly created transactions to the node's mempool and * prompt rebroadcasts (see ResendWalletTransactions()). */ bool fBroadcastTransactions = false; @@ -348,6 +348,8 @@ private: */ static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings); + static int64_t GetDefaultNextResend(); + public: /** * Main wallet lock. @@ -399,7 +401,6 @@ public: TxItems wtxOrdered; int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; - uint64_t nAccountingEntryNumber = 0; std::map<CTxDestination, CAddressBookData> m_address_book GUARDED_BY(cs_wallet); const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -537,6 +538,10 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress); void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; + /** Set the next time this wallet should resend transactions to 12-36 hours from now, ~1 day on average. */ + void SetNextResend() { m_next_resend = GetDefaultNextResend(); } + /** Return true if all conditions for periodically resending transactions are met. */ + bool ShouldResend() const; void ResubmitWalletTransactions(bool relay, bool force); OutputType TransactionChangeType(const std::optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) const; diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index fa3944e32b..97c2599366 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H - +#include <cstdint> #include <memory> #include <string> diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index b9b7019a3c..6ee134f392 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -3,13 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <zmq/zmqnotificationinterface.h> + +#include <logging.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <util/system.h> +#include <validationinterface.h> +#include <zmq/zmqabstractnotifier.h> #include <zmq/zmqpublishnotifier.h> #include <zmq/zmqutil.h> #include <zmq.h> -#include <primitives/block.h> -#include <util/system.h> +#include <cassert> +#include <map> +#include <string> +#include <utility> +#include <vector> CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(nullptr) { diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 8f81bfd63f..585e900ca6 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -5,10 +5,14 @@ #ifndef BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H #define BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H +#include <primitives/transaction.h> #include <validationinterface.h> + +#include <cstdint> #include <list> #include <memory> +class CBlock; class CBlockIndex; class CZMQAbstractNotifier; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 51c8ad515e..eaf3455296 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -6,21 +6,37 @@ #include <chain.h> #include <chainparams.h> +#include <crypto/common.h> +#include <logging.h> +#include <netaddress.h> #include <netbase.h> #include <node/blockstorage.h> +#include <primitives/block.h> +#include <primitives/transaction.h> #include <rpc/server.h> +#include <serialize.h> #include <streams.h> -#include <util/system.h> +#include <sync.h> +#include <uint256.h> +#include <version.h> #include <zmq/zmqutil.h> #include <zmq.h> +#include <cassert> #include <cstdarg> #include <cstddef> +#include <cstdint> +#include <cstring> #include <map> #include <optional> #include <string> #include <utility> +#include <vector> + +namespace Consensus { +struct Params; +} using node::ReadBlockFromDisk; diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index c1d66bddb1..fcedd1aabe 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -7,7 +7,11 @@ #include <zmq/zmqabstractnotifier.h> +#include <cstddef> +#include <cstdint> + class CBlockIndex; +class CTransaction; class CZMQAbstractPublishNotifier : public CZMQAbstractNotifier { diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index ec6d1cbba3..047e6bf9b7 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -11,6 +11,11 @@ #include <univalue.h> +#include <list> +#include <string> + +class JSONRPCRequest; + namespace { static RPCHelpMan getzmqnotifications() diff --git a/test/functional/data/rpc_psbt.json b/test/functional/data/rpc_psbt.json index 657faebffc..3127350872 100644 --- a/test/functional/data/rpc_psbt.json +++ b/test/functional/data/rpc_psbt.json @@ -44,6 +44,10 @@ [ "cHNidP8BAKOro2MDAwMDA5ggCAAA////CQAtAAD+///1AAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJAAAAAAAAAAAAAAAAAAAAAAAAAD+///1Zm9ybmV3nWx1Y2vmelLmegAAAAAAAAAAAAAAAAAAAAMKAwMDAwMDAwMDAwMACvMBA3FkAAAAAAAAAAAABAAlAAAAAAAAACEWDQ0zDQ0NDQ0NDQ0NCwEAAH9/f39/fwMAAABNo6P///kAAA==", "Input Taproot BIP32 keypath has an invalid length" + ], + [ + "cHNidP8BAIkCAAAAAapfm08b0MipBvW9thL06f8rMbeazW7TIa0W9plHj4WoAAAAAAD9////AoCWmAAAAAAAIlEgC+blBlIP1iijRWxqjw1u9H02sqr7y8fno6/LdnvGqPl895x2AAAAACJRIM5wyjSexMbADl4K+AI1/68zyaDlE7guKvrEDUAjwqU1AAAAAAABASsAlDV3AAAAACJRIDfCpO/CIAqc0JKgBhsCfaPGdyroYtmH+4gQK/Mnn72UIRZGOixxmh9h2gqDIecYHcQHRa8w+Sokc//iDiqXz7uMGRkAHzYIzlYAAIABAACAAAAAgAAAAABhAAAAARcgRjoscZofYdoKgyHnGB3EB0WvMPkqJHP/4g4ql8+7jBkAAQUg1YCB33LpmkGemw3ncz7fcnjhL/bBG/PjH8vpgr2L3cUBBgAhB9WAgd9y6ZpBnpsN53M+33J44S/2wRvz4x/L6YK9i93FGQAfNgjOVgAAgAEAAIAAAACAAAAAAGIAAAAAAQUg9jMNus8cd+GAosBk9wn+pNP9wn7A+jy2Vq0cy+siJ8wBBgAhB/YzDbrPHHfhgKLAZPcJ/qTT/cJ+wPo8tlatHMvrIifMGQAfNgjOVgAAgAEAAIAAAACAAQAAAFEBAAAA", + "Output Taproot tree must not be empty" ] ], "valid" : [ diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 05d274a9fe..5b43fe4f8e 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -10,15 +10,21 @@ from test_framework.blocktools import ( NORMAL_GBT_REQUEST_PARAMS, add_witness_commitment, create_block, + script_to_p2wsh_script, ) from test_framework.messages import ( COIN, COutPoint, CTransaction, CTxIn, + CTxInWitness, CTxOut, tx_from_hex, ) +from test_framework.script import ( + CScript, + OP_TRUE, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -26,7 +32,8 @@ from test_framework.util import ( assert_raises_rpc_error, softfork_active, ) -from test_framework.script_util import DUMMY_P2WPKH_SCRIPT + +SCRIPT_W0_SH_OP_TRUE = script_to_p2wsh_script(CScript([OP_TRUE])) SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) SEQUENCE_LOCKTIME_TYPE_FLAG = (1<<22) # this means use time (0 means height) @@ -42,11 +49,9 @@ class BIP68Test(BitcoinTestFramework): self.extra_args = [ [ '-testactivationheight=csv@432', - "-acceptnonstdtxn=1", ], [ '-testactivationheight=csv@432', - "-acceptnonstdtxn=0", ], ] @@ -100,7 +105,7 @@ class BIP68Test(BitcoinTestFramework): # input to mature. sequence_value = SEQUENCE_LOCKTIME_DISABLE_FLAG | 1 tx1.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), nSequence=sequence_value)] - tx1.vout = [CTxOut(value, DUMMY_P2WPKH_SCRIPT)] + tx1.vout = [CTxOut(value, SCRIPT_W0_SH_OP_TRUE)] tx1_signed = self.nodes[0].signrawtransactionwithwallet(tx1.serialize().hex())["hex"] tx1_id = self.nodes[0].sendrawtransaction(tx1_signed) @@ -112,7 +117,9 @@ class BIP68Test(BitcoinTestFramework): tx2.nVersion = 2 sequence_value = sequence_value & 0x7fffffff tx2.vin = [CTxIn(COutPoint(tx1_id, 0), nSequence=sequence_value)] - tx2.vout = [CTxOut(int(value - self.relayfee * COIN), DUMMY_P2WPKH_SCRIPT)] + tx2.wit.vtxinwit = [CTxInWitness()] + tx2.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx2.vout = [CTxOut(int(value - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx2.rehash() assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx2.serialize().hex()) @@ -207,7 +214,7 @@ class BIP68Test(BitcoinTestFramework): value += utxos[j]["amount"]*COIN # Overestimate the size of the tx - signatures should be less than 120 bytes, and leave 50 for the output tx_size = len(tx.serialize().hex())//2 + 120*num_inputs + 50 - tx.vout.append(CTxOut(int(value-self.relayfee*tx_size*COIN/1000), DUMMY_P2WPKH_SCRIPT)) + tx.vout.append(CTxOut(int(value - self.relayfee * tx_size * COIN / 1000), SCRIPT_W0_SH_OP_TRUE)) rawtx = self.nodes[0].signrawtransactionwithwallet(tx.serialize().hex())["hex"] if (using_sequence_locks and not should_pass): @@ -236,7 +243,7 @@ class BIP68Test(BitcoinTestFramework): tx2 = CTransaction() tx2.nVersion = 2 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] - tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee*COIN), DUMMY_P2WPKH_SCRIPT)] + tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] tx2 = tx_from_hex(tx2_raw) tx2.rehash() @@ -254,7 +261,9 @@ class BIP68Test(BitcoinTestFramework): tx = CTransaction() tx.nVersion = 2 tx.vin = [CTxIn(COutPoint(orig_tx.sha256, 0), nSequence=sequence_value)] - tx.vout = [CTxOut(int(orig_tx.vout[0].nValue - relayfee * COIN), DUMMY_P2WPKH_SCRIPT)] + tx.wit.vtxinwit = [CTxInWitness()] + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx.vout = [CTxOut(int(orig_tx.vout[0].nValue - relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx.rehash() if (orig_tx.hash in node.getrawmempool()): @@ -367,7 +376,7 @@ class BIP68Test(BitcoinTestFramework): tx2 = CTransaction() tx2.nVersion = 1 tx2.vin = [CTxIn(COutPoint(tx1.sha256, 0), nSequence=0)] - tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee*COIN), DUMMY_P2WPKH_SCRIPT)] + tx2.vout = [CTxOut(int(tx1.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] # sign tx2 tx2_raw = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())["hex"] @@ -382,7 +391,9 @@ class BIP68Test(BitcoinTestFramework): tx3 = CTransaction() tx3.nVersion = 2 tx3.vin = [CTxIn(COutPoint(tx2.sha256, 0), nSequence=sequence_value)] - tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee * COIN), DUMMY_P2WPKH_SCRIPT)] + tx3.wit.vtxinwit = [CTxInWitness()] + tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx3.vout = [CTxOut(int(tx2.vout[0].nValue - self.relayfee * COIN), SCRIPT_W0_SH_OP_TRUE)] tx3.rehash() assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx3.serialize().hex()) diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index eb31bca29a..112dbb9e6a 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -20,11 +20,25 @@ class ConfArgsTest(BitcoinTestFramework): self.disable_autoconnect = False def test_config_file_parser(self): + self.log.info('Test config file parser') self.stop_node(0) + # Check that startup fails if conf= is set in bitcoin.conf or in an included conf file + bad_conf_file_path = os.path.join(self.options.tmpdir, 'node0', 'bitcoin_bad.conf') + util.write_config(bad_conf_file_path, n=0, chain='', extra_config=f'conf=some.conf\n') + conf_in_config_file_err = 'Error: Error reading configuration file: conf cannot be set in the configuration file; use includeconf= if you want to include additional config files' + self.nodes[0].assert_start_raises_init_error( + extra_args=[f'-conf={bad_conf_file_path}'], + expected_msg=conf_in_config_file_err, + ) inc_conf_file_path = os.path.join(self.nodes[0].datadir, 'include.conf') with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: conf.write(f'includeconf={inc_conf_file_path}\n') + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('conf=some.conf\n') + self.nodes[0].assert_start_raises_init_error( + expected_msg=conf_in_config_file_err, + ) self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Error parsing command line arguments: Invalid parameter -dash_cli=1', @@ -32,11 +46,19 @@ class ConfArgsTest(BitcoinTestFramework): ) with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('dash_conf=1\n') + with self.nodes[0].assert_debug_log(expected_msgs=['Ignoring unknown configuration value dash_conf']): self.start_node(0) self.stop_node(0) with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: + conf.write('reindex=1\n') + + with self.nodes[0].assert_debug_log(expected_msgs=['Warning: reindex=1 is set in the configuration file, which will significantly slow down startup. Consider removing or commenting out this option for better performance, unless there is currently a condition which makes rebuilding the indexes necessary']): + self.start_node(0) + self.stop_node(0) + + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('-dash=1\n') self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Error reading configuration file: parse error on line 1: -dash=1, options in configuration file must be specified without leading -') diff --git a/test/functional/feature_discover.py b/test/functional/feature_discover.py new file mode 100755 index 0000000000..7f4b81114e --- /dev/null +++ b/test/functional/feature_discover.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test -discover command.""" + +import socket + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +def is_valid_ipv4_address(address): + try: + socket.inet_aton(address) + except socket.error: + return False + return True + + +def is_valid_ipv6_address(address): + try: + socket.inet_pton(socket.AF_INET6, address) + except socket.error: + return False + return True + + +class DiscoverTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.bind_to_localhost_only = False + self.num_nodes = 1 + + def validate_addresses(self, addresses_obj): + for address_obj in addresses_obj: + address = address_obj['address'] + self.log.info(f"Validating {address}") + valid = (is_valid_ipv4_address(address) + or is_valid_ipv6_address(address)) + assert_equal(valid, True) + + def test_local_addresses(self, test_case, *, expect_empty=False): + self.log.info(f"Restart node with {test_case}") + self.restart_node(0, test_case) + network_info = self.nodes[0].getnetworkinfo() + network_enabled = [n for n in network_info['networks'] + if n['reachable'] and n['name'] in ['ipv4', 'ipv6']] + local_addrs = list(network_info["localaddresses"]) + if expect_empty or not network_enabled: + assert_equal(local_addrs, []) + elif len(local_addrs) > 0: + self.validate_addresses(local_addrs) + + def run_test(self): + test_cases = [ + ["-listen", "-discover"], + ["-discover"], + ] + + test_cases_empty = [ + ["-discover=0"], + ["-listen", "-discover=0"], + [], + ] + + for test_case in test_cases: + self.test_local_addresses(test_case, expect_empty=False) + + for test_case in test_cases_empty: + self.test_local_addresses(test_case, expect_empty=True) + + +if __name__ == '__main__': + DiscoverTest().main() diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py index 13c7326519..56d093c396 100755 --- a/test/functional/feature_init.py +++ b/test/functional/feature_init.py @@ -55,7 +55,6 @@ class InitStressTest(BitcoinTestFramework): b'Loading P2P addresses', b'Loading banlist', b'Loading block index', - b'Switching active chainstate', b'Checking all blk files are present', b'Loaded best chain:', b'init message: Verifying blocks', diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index c90852562e..18b079cd71 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -317,19 +317,34 @@ class ProxyTest(BitcoinTestFramework): self.stop_node(1) - self.log.info("Test passing invalid -proxy raises expected init error") - self.nodes[1].extra_args = ["-proxy=abc:def"] - msg = "Error: Invalid -proxy address or hostname: 'abc:def'" + self.log.info("Test passing invalid -proxy hostname raises expected init error") + self.nodes[1].extra_args = ["-proxy=abc..abc:23456"] + msg = "Error: Invalid -proxy address or hostname: 'abc..abc:23456'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) - self.log.info("Test passing invalid -onion raises expected init error") - self.nodes[1].extra_args = ["-onion=xyz:abc"] - msg = "Error: Invalid -onion address or hostname: 'xyz:abc'" + self.log.info("Test passing invalid -proxy port raises expected init error") + self.nodes[1].extra_args = ["-proxy=192.0.0.1:def"] + msg = "Error: Invalid port specified in -proxy: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) - self.log.info("Test passing invalid -i2psam raises expected init error") - self.nodes[1].extra_args = ["-i2psam=def:xyz"] - msg = "Error: Invalid -i2psam address or hostname: 'def:xyz'" + self.log.info("Test passing invalid -onion hostname raises expected init error") + self.nodes[1].extra_args = ["-onion=xyz..xyz:23456"] + msg = "Error: Invalid -onion address or hostname: 'xyz..xyz:23456'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -onion port raises expected init error") + self.nodes[1].extra_args = ["-onion=192.0.0.1:def"] + msg = "Error: Invalid port specified in -onion: '192.0.0.1:def'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam hostname raises expected init error") + self.nodes[1].extra_args = ["-i2psam=def..def:23456"] + msg = "Error: Invalid -i2psam address or hostname: 'def..def:23456'" + self.nodes[1].assert_start_raises_init_error(expected_msg=msg) + + self.log.info("Test passing invalid -i2psam port raises expected init error") + self.nodes[1].extra_args = ["-i2psam=192.0.0.1:def"] + msg = "Error: Invalid port specified in -i2psam: '192.0.0.1:def'" self.nodes[1].assert_start_raises_init_error(expected_msg=msg) self.log.info("Test passing invalid -onlynet=i2p without -i2psam raises expected init error") diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 7dbeccbc09..4126ffcd51 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -358,6 +358,8 @@ class PruneTest(BitcoinTestFramework): self.restart_node(2, extra_args=["-prune=550"]) self.log.info("Success") + assert_raises_rpc_error(-4, "Importing wallets is disabled when blocks are pruned", self.nodes[2].importwallet, "abc") + # check that wallet loads successfully when restarting a pruned node after IBD. # this was reported to fail in #7494. self.log.info("Syncing node 5 to test wallet") diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 610a28c56b..24252610be 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -387,6 +387,17 @@ class RESTTest (BitcoinTestFramework): assert_equal(self.test_rest_request(f"/headers/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/headers/1/{bb_hash}")) assert_equal(self.test_rest_request(f"/blockfilterheaders/basic/{bb_hash}", query_params={"count": 1}), self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")) + self.log.info("Test the /deploymentinfo URI") + + deployment_info = self.nodes[0].getdeploymentinfo() + assert_equal(deployment_info, self.test_rest_request('/deploymentinfo')) + + non_existing_blockhash = '42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4' + resp = self.test_rest_request(f'/deploymentinfo/{non_existing_blockhash}', ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), "Block not found") + + resp = self.test_rest_request(f"/deploymentinfo/{INVALID_PARAM}", ret_type=RetType.OBJ, status=400) + assert_equal(resp.read().decode('utf-8').rstrip(), f"Invalid hash: {INVALID_PARAM}") if __name__ == '__main__': RESTTest().main() diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 9c64bb1945..ac7eb96ac1 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -200,7 +200,7 @@ class MiningTest(BitcoinTestFramework): self.log.info("getblocktemplate: Test bad timestamps") bad_block = copy.deepcopy(block) - bad_block.nTime = 2**31 - 1 + bad_block.nTime = 2**32 - 1 assert_template(node, bad_block, 'time-too-new') assert_submitblock(bad_block, 'time-too-new', 'time-too-new') bad_block.nTime = 0 diff --git a/test/functional/p2p_sendtxrcncl.py b/test/functional/p2p_sendtxrcncl.py new file mode 100755 index 0000000000..f4c5dd4586 --- /dev/null +++ b/test/functional/p2p_sendtxrcncl.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test SENDTXRCNCL message +""" + +from test_framework.messages import ( + msg_sendtxrcncl, + msg_verack, + msg_version, + msg_wtxidrelay, +) +from test_framework.p2p import ( + P2PInterface, + P2P_SERVICES, + P2P_SUBVERSION, + P2P_VERSION, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class PeerNoVerack(P2PInterface): + def __init__(self, wtxidrelay=True): + super().__init__(wtxidrelay=wtxidrelay) + + def on_version(self, message): + # Avoid sending verack in response to version. + # When calling add_p2p_connection, wait_for_verack=False must be set (see + # comment in add_p2p_connection). + if message.nVersion >= 70016 and self.wtxidrelay: + self.send_message(msg_wtxidrelay()) + +class SendTxrcnclReceiver(P2PInterface): + def __init__(self): + super().__init__() + self.sendtxrcncl_msg_received = None + + def on_sendtxrcncl(self, message): + self.sendtxrcncl_msg_received = message + +class PeerTrackMsgOrder(P2PInterface): + def __init__(self): + super().__init__() + self.messages = [] + + def on_message(self, message): + super().on_message(message) + self.messages.append(message) + +def create_sendtxrcncl_msg(initiator=True): + sendtxrcncl_msg = msg_sendtxrcncl() + sendtxrcncl_msg.initiator = initiator + sendtxrcncl_msg.responder = not initiator + sendtxrcncl_msg.version = 1 + sendtxrcncl_msg.salt = 2 + return sendtxrcncl_msg + +class SendTxRcnclTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [['-txreconciliation']] + + def run_test(self): + self.log.info('SENDTXRCNCL sent to an inbound') + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=True, wait_for_verack=True) + assert peer.sendtxrcncl_msg_received + assert not peer.sendtxrcncl_msg_received.initiator + assert peer.sendtxrcncl_msg_received.responder + assert_equal(peer.sendtxrcncl_msg_received.version, 1) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL should be sent before VERACK') + peer = self.nodes[0].add_p2p_connection(PeerTrackMsgOrder(), send_version=True, wait_for_verack=True) + peer.wait_for_verack() + verack_index = [i for i, msg in enumerate(peer.messages) if msg.msgtype == b'verack'][0] + sendtxrcncl_index = [i for i, msg in enumerate(peer.messages) if msg.msgtype == b'sendtxrcncl'][0] + assert(sendtxrcncl_index < verack_index) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL on pre-WTXID version should not be sent') + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=False, wait_for_verack=False) + pre_wtxid_version_msg = msg_version() + pre_wtxid_version_msg.nVersion = 70015 + pre_wtxid_version_msg.strSubVer = P2P_SUBVERSION + pre_wtxid_version_msg.nServices = P2P_SERVICES + pre_wtxid_version_msg.relay = 1 + peer.send_message(pre_wtxid_version_msg) + peer.wait_for_verack() + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL for fRelay=false should not be sent') + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=False, wait_for_verack=False) + no_txrelay_version_msg = msg_version() + no_txrelay_version_msg.nVersion = P2P_VERSION + no_txrelay_version_msg.strSubVer = P2P_SUBVERSION + no_txrelay_version_msg.nServices = P2P_SERVICES + no_txrelay_version_msg.relay = 0 + peer.send_message(no_txrelay_version_msg) + peer.wait_for_verack() + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('valid SENDTXRCNCL received') + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(create_sendtxrcncl_msg()) + self.wait_until(lambda : "sendtxrcncl" in self.nodes[0].getpeerinfo()[-1]["bytesrecv_per_msg"]) + self.log.info('second SENDTXRCNCL triggers a disconnect') + peer.send_message(create_sendtxrcncl_msg()) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with initiator=responder=0 triggers a disconnect') + sendtxrcncl_no_role = create_sendtxrcncl_msg() + sendtxrcncl_no_role.initiator = False + sendtxrcncl_no_role.responder = False + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(sendtxrcncl_no_role) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with initiator=0 and responder=1 from inbound triggers a disconnect') + sendtxrcncl_wrong_role = create_sendtxrcncl_msg(initiator=False) + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(sendtxrcncl_wrong_role) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with version=0 triggers a disconnect') + sendtxrcncl_low_version = create_sendtxrcncl_msg() + sendtxrcncl_low_version.version = 0 + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_message(sendtxrcncl_low_version) + peer.wait_for_disconnect() + + self.log.info('sending SENDTXRCNCL after sending VERACK triggers a disconnect') + # We use PeerNoVerack even though verack is sent right after, to make sure it was actually + # sent before sendtxrcncl is sent. + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(), send_version=True, wait_for_verack=False) + peer.send_and_ping(msg_verack()) + peer.send_message(create_sendtxrcncl_msg()) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL without WTXIDRELAY is ignored (recon state is erased after VERACK)') + peer = self.nodes[0].add_p2p_connection(PeerNoVerack(wtxidrelay=False), send_version=True, wait_for_verack=False) + with self.nodes[0].assert_debug_log(['Forget txreconciliation state of peer']): + peer.send_message(create_sendtxrcncl_msg()) + peer.send_message(msg_verack()) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL sent to an outbound') + peer = self.nodes[0].add_outbound_p2p_connection( + SendTxrcnclReceiver(), wait_for_verack=True, p2p_idx=1, connection_type="outbound-full-relay") + assert peer.sendtxrcncl_msg_received + assert peer.sendtxrcncl_msg_received.initiator + assert not peer.sendtxrcncl_msg_received.responder + assert_equal(peer.sendtxrcncl_msg_received.version, 1) + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL should not be sent if block-relay-only') + peer = self.nodes[0].add_outbound_p2p_connection( + SendTxrcnclReceiver(), wait_for_verack=True, p2p_idx=2, connection_type="block-relay-only") + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL if block-relay-only triggers a disconnect') + peer = self.nodes[0].add_outbound_p2p_connection( + PeerNoVerack(), wait_for_verack=False, p2p_idx=3, connection_type="block-relay-only") + peer.send_message(create_sendtxrcncl_msg(initiator=False)) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL with initiator=1 and responder=0 from outbound triggers a disconnect') + sendtxrcncl_wrong_role = create_sendtxrcncl_msg(initiator=True) + peer = self.nodes[0].add_outbound_p2p_connection( + P2PInterface(), wait_for_verack=False, p2p_idx=4, connection_type="outbound-full-relay") + peer.send_message(sendtxrcncl_wrong_role) + peer.wait_for_disconnect() + + self.log.info('SENDTXRCNCL not sent if -txreconciliation flag is not set') + self.restart_node(0, []) + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=True, wait_for_verack=True) + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + self.log.info('SENDTXRCNCL not sent if blocksonly is set') + self.restart_node(0, ["-txreconciliation", "-blocksonly"]) + peer = self.nodes[0].add_p2p_connection(SendTxrcnclReceiver(), send_version=True, wait_for_verack=True) + assert not peer.sendtxrcncl_msg_received + peer.peer_disconnect() + + +if __name__ == '__main__': + SendTxRcnclTest().main() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index d07d28879e..80e8fe55a3 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -89,6 +89,7 @@ class BlockchainTest(BitcoinTestFramework): self._test_waitforblockheight() self._test_getblock() self._test_getdeploymentinfo() + self._test_y2106() assert self.nodes[0].verifychain(4, 0) def mine_chain(self): @@ -255,6 +256,14 @@ class BlockchainTest(BitcoinTestFramework): # calling with an explicit hash works self.check_signalling_deploymentinfo_result(self.nodes[0].getdeploymentinfo(gbci207["bestblockhash"]), gbci207["blocks"], gbci207["bestblockhash"], "started") + def _test_y2106(self): + self.log.info("Check that block timestamps work until year 2106") + self.generate(self.nodes[0], 8)[-1] + time_2106 = 2**32 - 1 + self.nodes[0].setmocktime(time_2106) + last = self.generate(self.nodes[0], 6)[-1] + assert_equal(self.nodes[0].getblockheader(last)["mediantime"], time_2106) + def _test_getchaintxstats(self): self.log.info("Test getchaintxstats") diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 4583ca25cf..3b78a7d095 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -27,7 +27,10 @@ from test_framework.psbt import ( PSBT_IN_SHA256, PSBT_IN_HASH160, PSBT_IN_HASH256, + PSBT_IN_WITNESS_UTXO, + PSBT_OUT_TAP_TREE, ) +from test_framework.script import CScript, OP_TRUE from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -779,9 +782,18 @@ class PSBTTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) self.nodes[0].importdescriptors([{"desc": descsum_create("tr({})".format(privkey)), "timestamp":"now"}]) - psbt = watchonly.sendall([wallet.getnewaddress()])["psbt"] + psbt = watchonly.sendall([wallet.getnewaddress(), addr])["psbt"] psbt = self.nodes[0].walletprocesspsbt(psbt)["psbt"] - self.nodes[0].sendrawtransaction(self.nodes[0].finalizepsbt(psbt)["hex"]) + txid = self.nodes[0].sendrawtransaction(self.nodes[0].finalizepsbt(psbt)["hex"]) + vout = find_vout_for_address(self.nodes[0], txid, addr) + + # Make sure tap tree is in psbt + parsed_psbt = PSBT.from_base64(psbt) + assert_greater_than(len(parsed_psbt.o[vout].map[PSBT_OUT_TAP_TREE]), 0) + assert "taproot_tree" in self.nodes[0].decodepsbt(psbt)["outputs"][vout] + parsed_psbt.make_blank() + comb_psbt = self.nodes[0].combinepsbt([psbt, parsed_psbt.to_base64()]) + assert_equal(comb_psbt, psbt) self.log.info("Test that walletprocesspsbt both updates and signs a non-updated psbt containing Taproot inputs") addr = self.nodes[0].getnewaddress("", "bech32m") @@ -793,6 +805,14 @@ class PSBTTest(BitcoinTestFramework): self.nodes[0].sendrawtransaction(rawtx) self.generate(self.nodes[0], 1) + # Make sure tap tree is not in psbt + parsed_psbt = PSBT.from_base64(psbt) + assert PSBT_OUT_TAP_TREE not in parsed_psbt.o[0].map + assert "taproot_tree" not in self.nodes[0].decodepsbt(psbt)["outputs"][0] + parsed_psbt.make_blank() + comb_psbt = self.nodes[0].combinepsbt([psbt, parsed_psbt.to_base64()]) + assert_equal(comb_psbt, psbt) + self.log.info("Test decoding PSBT with per-input preimage types") # note that the decodepsbt RPC doesn't check whether preimages and hashes match hash_ripemd160, preimage_ripemd160 = random_bytes(20), random_bytes(50) @@ -834,6 +854,18 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "PSBTs not compatible (different transactions)", self.nodes[0].combinepsbt, [psbt1, psbt2]) assert_equal(self.nodes[0].combinepsbt([psbt1, psbt1]), psbt1) + self.log.info("Test that PSBT inputs are being checked via script execution") + acs_prevout = CTxOut(nValue=0, scriptPubKey=CScript([OP_TRUE])) + tx = CTransaction() + tx.vin = [CTxIn(outpoint=COutPoint(hash=int('dd' * 32, 16), n=0), scriptSig=b"")] + tx.vout = [CTxOut(nValue=0, scriptPubKey=b"")] + psbt = PSBT() + psbt.g = PSBTMap({PSBT_GLOBAL_UNSIGNED_TX: tx.serialize()}) + psbt.i = [PSBTMap({bytes([PSBT_IN_WITNESS_UTXO]) : acs_prevout.serialize()})] + psbt.o = [PSBTMap()] + assert_equal(self.nodes[0].finalizepsbt(psbt.to_base64()), + {'hex': '0200000001dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd0000000000000000000100000000000000000000000000', 'complete': True}) + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_scanblocks.py b/test/functional/rpc_scanblocks.py new file mode 100755 index 0000000000..39f091fd1a --- /dev/null +++ b/test/functional/rpc_scanblocks.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the scanblocks RPC call.""" +from test_framework.messages import COIN +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.wallet import ( + MiniWallet, + getnewdestination, +) + + +class ScanblocksTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.extra_args = [["-blockfilterindex=1"], []] + + def run_test(self): + node = self.nodes[0] + wallet = MiniWallet(node) + wallet.rescan_utxos() + + # send 1.0, mempool only + _, spk_1, addr_1 = getnewdestination() + wallet.send_to(from_node=node, scriptPubKey=spk_1, amount=1 * COIN) + + parent_key = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" + # send 1.0, mempool only + # childkey 5 of `parent_key` + wallet.send_to(from_node=node, + scriptPubKey=bytes.fromhex(node.validateaddress("mkS4HXoTYWRTescLGaUTGbtTTYX5EjJyEE")['scriptPubKey']), + amount=1 * COIN) + + # mine a block and assure that the mined blockhash is in the filterresult + blockhash = self.generate(node, 1)[0] + height = node.getblockheader(blockhash)['height'] + self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) + + out = node.scanblocks("start", [f"addr({addr_1})"]) + assert(blockhash in out['relevant_blocks']) + assert_equal(height, out['to_height']) + assert_equal(0, out['from_height']) + + # mine another block + blockhash_new = self.generate(node, 1)[0] + height_new = node.getblockheader(blockhash_new)['height'] + + # make sure the blockhash is not in the filter result if we set the start_height + # to the just mined block (unlikely to hit a false positive) + assert(blockhash not in node.scanblocks( + "start", [f"addr({addr_1})"], height_new)['relevant_blocks']) + + # make sure the blockhash is present when using the first mined block as start_height + assert(blockhash in node.scanblocks( + "start", [f"addr({addr_1})"], height)['relevant_blocks']) + + # also test the stop height + assert(blockhash in node.scanblocks( + "start", [f"addr({addr_1})"], height, height)['relevant_blocks']) + + # use the stop_height to exclude the relevant block + assert(blockhash not in node.scanblocks( + "start", [f"addr({addr_1})"], 0, height - 1)['relevant_blocks']) + + # make sure the blockhash is present when using the first mined block as start_height + assert(blockhash in node.scanblocks( + "start", [{"desc": f"pkh({parent_key}/*)", "range": [0, 100]}], height)['relevant_blocks']) + + # test node with disabled blockfilterindex + assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", + self.nodes[1].scanblocks, "start", [f"addr({addr_1})"]) + + # test unknown filtertype + assert_raises_rpc_error(-5, "Unknown filtertype", + node.scanblocks, "start", [f"addr({addr_1})"], 0, 10, "extended") + + # test invalid start_height + assert_raises_rpc_error(-1, "Invalid start_height", + node.scanblocks, "start", [f"addr({addr_1})"], 100000000) + + # test invalid stop_height + assert_raises_rpc_error(-1, "Invalid stop_height", + node.scanblocks, "start", [f"addr({addr_1})"], 10, 0) + assert_raises_rpc_error(-1, "Invalid stop_height", + node.scanblocks, "start", [f"addr({addr_1})"], 10, 100000000) + + # test accessing the status (must be empty) + assert_equal(node.scanblocks("status"), None) + + # test aborting the current scan (there is no, must return false) + assert_equal(node.scanblocks("abort"), False) + + # test invalid command + assert_raises_rpc_error(-8, "Invalid command", node.scanblocks, "foobar") + + +if __name__ == '__main__': + ScanblocksTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index acb6d3ea4a..6eb5b493b9 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -33,6 +33,9 @@ class ScantxoutsetTest(BitcoinTestFramework): self.wallet = MiniWallet(self.nodes[0]) self.wallet.rescan_utxos() + self.log.info("Test if we find coinbase outputs.") + assert_equal(sum(u["coinbase"] for u in self.nodes[0].scantxoutset("start", [self.wallet.get_descriptor()])["unspents"]), 49) + self.log.info("Create UTXOs...") pubk1, spk_P2SH_SEGWIT, addr_P2SH_SEGWIT = getnewdestination("p2sh-segwit") pubk2, spk_LEGACY, addr_LEGACY = getnewdestination("legacy") diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 8a928a1e50..252b49cc6d 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1838,3 +1838,31 @@ class msg_cfcheckpt: def __repr__(self): return "msg_cfcheckpt(filter_type={:#x}, stop_hash={:x})".format( self.filter_type, self.stop_hash) + +class msg_sendtxrcncl: + __slots__ = ("initiator", "responder", "version", "salt") + msgtype = b"sendtxrcncl" + + def __init__(self): + self.initiator = False + self.responder = False + self.version = 0 + self.salt = 0 + + def deserialize(self, f): + self.initiator = struct.unpack("<?", f.read(1))[0] + self.responder = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<I", f.read(4))[0] + self.salt = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.initiator) + r += struct.pack("<?", self.responder) + r += struct.pack("<I", self.version) + r += struct.pack("<Q", self.salt) + return r + + def __repr__(self): + return "msg_sendtxrcncl(initiator=%i, responder=%i, version=%lu, salt=%lu)" %\ + (self.initiator, self.responder, self.version, self.salt) diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index e96cc3fa7f..05b46e630c 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -62,6 +62,7 @@ from test_framework.messages import ( msg_sendaddrv2, msg_sendcmpct, msg_sendheaders, + msg_sendtxrcncl, msg_tx, MSG_TX, MSG_TYPE_MASK, @@ -126,6 +127,7 @@ MESSAGEMAP = { b"sendaddrv2": msg_sendaddrv2, b"sendcmpct": msg_sendcmpct, b"sendheaders": msg_sendheaders, + b"sendtxrcncl": msg_sendtxrcncl, b"tx": msg_tx, b"verack": msg_verack, b"version": msg_version, @@ -421,6 +423,7 @@ class P2PInterface(P2PConnection): def on_sendaddrv2(self, message): pass def on_sendcmpct(self, message): pass def on_sendheaders(self, message): pass + def on_sendtxrcncl(self, message): pass def on_tx(self, message): pass def on_wtxidrelay(self, message): pass diff --git a/test/functional/test_framework/psbt.py b/test/functional/test_framework/psbt.py index 68945e7e84..3a5b4ec74d 100644 --- a/test/functional/test_framework/psbt.py +++ b/test/functional/test_framework/psbt.py @@ -123,6 +123,15 @@ class PSBT: psbt = [x.serialize() for x in [self.g] + self.i + self.o] return b"psbt\xff" + b"".join(psbt) + def make_blank(self): + """ + Remove all fields except for PSBT_GLOBAL_UNSIGNED_TX + """ + for m in self.i + self.o: + m.map.clear() + + self.g = PSBTMap(map={0: self.g.map[0]}) + def to_base64(self): return base64.b64encode(self.serialize()).decode("utf8") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e35cae006f..2367a9a8fa 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -618,7 +618,7 @@ class TestNode(): return p2p_conn - def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs): + def add_outbound_p2p_connection(self, p2p_conn, *, wait_for_verack=True, p2p_idx, connection_type="outbound-full-relay", **kwargs): """Add an outbound p2p connection from node. Must be an "outbound-full-relay", "block-relay-only", "addr-fetch" or "feeler" connection. @@ -640,8 +640,9 @@ class TestNode(): p2p_conn.wait_for_connect() self.p2ps.append(p2p_conn) - p2p_conn.wait_for_verack() - p2p_conn.sync_with_ping() + if wait_for_verack: + p2p_conn.wait_for_verack() + p2p_conn.sync_with_ping() return p2p_conn diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index d78c1c634f..e20de8ea8e 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -277,6 +277,7 @@ BASE_SCRIPTS = [ 'feature_dersig.py', 'feature_cltv.py', 'rpc_uptime.py', + 'feature_discover.py', 'wallet_resendwallettransactions.py --legacy-wallet', 'wallet_resendwallettransactions.py --descriptors', 'wallet_fallbackfee.py --legacy-wallet', @@ -316,6 +317,8 @@ BASE_SCRIPTS = [ 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', + 'rpc_scanblocks.py', + 'p2p_sendtxrcncl.py', 'rpc_scantxoutset.py', 'feature_txindex_compatibility.py', 'feature_unsupported_utxo_db.py', diff --git a/test/functional/wallet_sendall.py b/test/functional/wallet_sendall.py index db4f32fe16..4fe11455b1 100755 --- a/test/functional/wallet_sendall.py +++ b/test/functional/wallet_sendall.py @@ -221,6 +221,11 @@ class SendallTest(BitcoinTestFramework): self.add_utxos([16, 5]) spent_utxo = self.wallet.listunspent()[0] + # fails on out of bounds vout + assert_raises_rpc_error(-8, + "Input not found. UTXO ({}:{}) is not part of wallet.".format(spent_utxo["txid"], 1000), + self.wallet.sendall, recipients=[self.remainder_target], options={"inputs": [{"txid": spent_utxo["txid"], "vout": 1000}]}) + # fails on unconfirmed spent UTXO self.wallet.sendall(recipients=[self.remainder_target]) assert_raises_rpc_error(-8, @@ -276,6 +281,33 @@ class SendallTest(BitcoinTestFramework): recipients=[self.remainder_target], fee_rate=100000) + @cleanup + def sendall_watchonly_specific_inputs(self): + self.log.info("Test sendall with a subset of UTXO pool in a watchonly wallet") + self.add_utxos([17, 4]) + utxo = self.wallet.listunspent()[0] + + self.nodes[0].createwallet(wallet_name="watching", disable_private_keys=True) + watchonly = self.nodes[0].get_wallet_rpc("watching") + + import_req = [{ + "desc": utxo["desc"], + "timestamp": 0, + }] + if self.options.descriptors: + watchonly.importdescriptors(import_req) + else: + watchonly.importmulti(import_req) + + sendall_tx_receipt = watchonly.sendall(recipients=[self.remainder_target], options={"inputs": [utxo]}) + psbt = sendall_tx_receipt["psbt"] + decoded = self.nodes[0].decodepsbt(psbt) + assert_equal(len(decoded["inputs"]), 1) + assert_equal(len(decoded["outputs"]), 1) + assert_equal(decoded["tx"]["vin"][0]["txid"], utxo["txid"]) + assert_equal(decoded["tx"]["vin"][0]["vout"], utxo["vout"]) + assert_equal(decoded["tx"]["vout"][0]["scriptPubKey"]["address"], self.remainder_target) + # This tests needs to be the last one otherwise @cleanup will fail with "Transaction too large" error def sendall_fails_with_transaction_too_large(self): self.log.info("Test that sendall fails if resulting transaction is too large") @@ -341,6 +373,9 @@ class SendallTest(BitcoinTestFramework): # Sendall fails when providing a fee that is too high self.sendall_fails_on_high_fee() + # Sendall succeeds with watchonly wallets spending specific UTXOs + self.sendall_watchonly_specific_inputs() + # Sendall fails when many inputs result to too large transaction self.sendall_fails_with_transaction_too_large() diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index a0f17ac119..abf38ca79b 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -14,6 +14,7 @@ import sys EXPECTED_CIRCULAR_DEPENDENCIES = ( "chainparamsbase -> util/system -> chainparamsbase", "node/blockstorage -> validation -> node/blockstorage", + "node/utxo_snapshot -> validation -> node/utxo_snapshot", "policy/fees -> txmempool -> policy/fees", "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", |